{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.1"},"colab":{"name":"2-advanced-cnn.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"BKE8lvAJ6E38"},"source":["# Advanced Convolutional Neural Networks (CNN) - 1\n","- Objective: try different structures of CNNs\n","- Note: examples are performed on **i5 7600 + gtx 1060 6GB **"]},{"cell_type":"markdown","metadata":{"id":"OQdJl5pH6E3_"},"source":["## Deeper Networks\n","- It is one of definite trends that CNN models submitted to ILSVRC are getting deeper and deeper\n","<img src=\"https://cdn-images-1.medium.com/max/756/1*Io-I-fOM0jGftDb_nne4KQ.png\" style=\"width: 600px\"/>\n","\n","<br>\n","- Deeper networks are able to learn more \"abstract\" representations by comprising more operations and nonlinearities \n","- However, they are difficult and costly to train and prone to overfitting\n","\n","<img src=\"http://cs231n.github.io/assets/cnnvis/filt1.jpeg\" style=\"width: 400px\"/>"]},{"cell_type":"code","metadata":{"id":"hsmtPujc6E4A","executionInfo":{"status":"ok","timestamp":1604772739043,"user_tz":420,"elapsed":582,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["import numpy as np\n","import matplotlib.pyplot as plt\n","\n","from tensorflow.keras.datasets import mnist\n","from tensorflow.keras.utils import to_categorical"],"execution_count":2,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"3aKMBwfB6E4E"},"source":["## Load Dataset\n","- MNIST Dataset\n","- Doc: https://keras.io/datasets/"]},{"cell_type":"code","metadata":{"id":"5zaCclhU6E4F","executionInfo":{"status":"ok","timestamp":1604772743877,"user_tz":420,"elapsed":1123,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"c9e40c0d-90f0-4ae2-d978-39a95551855d","colab":{"base_uri":"https://localhost:8080/"}},"source":["(X_train, y_train), (X_test, y_test) = mnist.load_data()"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n","11493376/11490434 [==============================] - 0s 0us/step\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ja2gP-R36E4I","executionInfo":{"status":"ok","timestamp":1604772743878,"user_tz":420,"elapsed":1121,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["# reshaping X data: (n, 28, 28) => (n, 28, 28, 1)\n","X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], X_train.shape[2], 1))\n","X_test = X_test.reshape((X_test.shape[0], X_test.shape[1], X_test.shape[2], 1))"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"id":"dlOtwbls6E4M","executionInfo":{"status":"ok","timestamp":1604772744645,"user_tz":420,"elapsed":397,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["# converting y data into categorical (one-hot encoding)\n","y_train = to_categorical(y_train)\n","y_test = to_categorical(y_test)"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"qwa-m-NG6E4Q","executionInfo":{"status":"ok","timestamp":1604772745224,"user_tz":420,"elapsed":500,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"20889040-1298-4193-a9ac-4dce2de24fd1","colab":{"base_uri":"https://localhost:8080/"}},"source":["print(X_train.shape)\n","print(X_test.shape)\n","print(y_train.shape)\n","print(y_test.shape)"],"execution_count":6,"outputs":[{"output_type":"stream","text":["(60000, 28, 28, 1)\n","(10000, 28, 28, 1)\n","(60000, 10)\n","(10000, 10)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"collapsed":true,"id":"c1bkbviJ6E4T"},"source":["## 0. Basic CNN model\n","- Basic CNN similar to what we made before\n","- One convolutional & pooling layer"]},{"cell_type":"code","metadata":{"id":"U9WG0JFe6E4U","executionInfo":{"status":"ok","timestamp":1604772752598,"user_tz":420,"elapsed":867,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["from tensorflow.keras.models import Sequential\n","from tensorflow.keras import optimizers\n","from tensorflow.keras.layers import Dense, Activation, Flatten, Conv2D, MaxPooling2D"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"cctAMUdw6E4X","executionInfo":{"status":"ok","timestamp":1604772753494,"user_tz":420,"elapsed":537,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def basic_cnn():\n","    model = Sequential()\n","    \n","    model.add(Conv2D(input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]), filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","\n","    # prior layer should be flattend to be connected to dense layers\n","    model.add(Flatten())\n","    # dense layer with 50 neurons\n","    model.add(Dense(50, activation = 'relu'))\n","    # final layer with 10 neurons to classify the instances\n","    model.add(Dense(10, activation = 'softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model    "],"execution_count":8,"outputs":[]},{"cell_type":"code","metadata":{"id":"I__8ymxc6E4Z","executionInfo":{"status":"ok","timestamp":1604772755292,"user_tz":420,"elapsed":787,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["model = basic_cnn()"],"execution_count":9,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"Hqf9nuaO6E4b","executionInfo":{"status":"ok","timestamp":1604772755546,"user_tz":420,"elapsed":399,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"6fc76085-e37a-4c4d-a739-5dc6c7266917","colab":{"base_uri":"https://localhost:8080/"}},"source":["model.summary()"],"execution_count":10,"outputs":[{"output_type":"stream","text":["Model: \"sequential\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","conv2d (Conv2D)              (None, 28, 28, 50)        500       \n","_________________________________________________________________\n","activation (Activation)      (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","max_pooling2d (MaxPooling2D) (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","flatten (Flatten)            (None, 9800)              0         \n","_________________________________________________________________\n","dense (Dense)                (None, 50)                490050    \n","_________________________________________________________________\n","dense_1 (Dense)              (None, 10)                510       \n","=================================================================\n","Total params: 491,060\n","Trainable params: 491,060\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"8n411Nk36E4d","executionInfo":{"status":"ok","timestamp":1604773101545,"user_tz":420,"elapsed":270737,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"93466fcf-4a8d-4629-c0b4-1c719b9475d1","colab":{"base_uri":"https://localhost:8080/"}},"source":["%%time\n","history = model.fit(X_train, y_train, batch_size = 256, validation_split = 0.2, epochs = 10, verbose = 1)"],"execution_count":12,"outputs":[{"output_type":"stream","text":["Epoch 1/10\n","188/188 [==============================] - 27s 144ms/step - loss: 0.0375 - accuracy: 0.9887 - val_loss: 0.0809 - val_accuracy: 0.9779\n","Epoch 2/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0240 - accuracy: 0.9934 - val_loss: 0.0783 - val_accuracy: 0.9778\n","Epoch 3/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0175 - accuracy: 0.9955 - val_loss: 0.0798 - val_accuracy: 0.9788\n","Epoch 4/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0133 - accuracy: 0.9967 - val_loss: 0.0838 - val_accuracy: 0.9778\n","Epoch 5/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0102 - accuracy: 0.9976 - val_loss: 0.0859 - val_accuracy: 0.9783\n","Epoch 6/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0078 - accuracy: 0.9984 - val_loss: 0.0871 - val_accuracy: 0.9779\n","Epoch 7/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0060 - accuracy: 0.9986 - val_loss: 0.0915 - val_accuracy: 0.9783\n","Epoch 8/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0044 - accuracy: 0.9993 - val_loss: 0.0928 - val_accuracy: 0.9785\n","Epoch 9/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0036 - accuracy: 0.9993 - val_loss: 0.0979 - val_accuracy: 0.9780\n","Epoch 10/10\n","188/188 [==============================] - 27s 143ms/step - loss: 0.0027 - accuracy: 0.9996 - val_loss: 0.0987 - val_accuracy: 0.9781\n","CPU times: user 8min 5s, sys: 8.68 s, total: 8min 13s\n","Wall time: 4min 30s\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Hq3kZBjT6E4g","executionInfo":{"status":"ok","timestamp":1604773128875,"user_tz":420,"elapsed":514,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"72dfffe5-a1ee-4c03-a0f4-6ac2f0dc1b60","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.legend(['training', 'validation'], loc = 'upper left')\n","plt.show()"],"execution_count":15,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dcnGyEhISFhDxLQKCCyRkVxwaUW1Lqgdbfiry2t1VLb2/6Kt71t9crt8rPW2qv2qqW9WpdSrVdtQbAtVK3LJcguyCaSQIAQCEkgIct8f3+ck2QIAQYYOEnO+/l4zCNnvmeZ7xmYz/tsc8acc4iISPgkBN0BEREJhgJARCSkFAAiIiGlABARCSkFgIhISCUF3YEjkZub6/Lz84PuhohIh7Jo0aIdzrmerds7VADk5+dTVFQUdDdERDoUM/u0rXYdAhIRCSkFgIhISMUUAGY208y2m9mKg4w3M3vUzNaZ2TIzGxM17g4zW+s/7ohqH2tmy/15HjUzO/bVERGRWMV6DuB3wH8Czxxk/CSgwH+cDTwBnG1mPYAfAoWAAxaZ2WvOuV3+NF8GPgBmAxOBOUe6AvX19ZSUlFBbW3uks0obUlNTycvLIzk5OeiuiMhxFlMAOOfeMrP8Q0xyNfCM824s9L6ZZZlZX2AC8KZzbieAmb0JTDSzBUCmc+59v/0Z4BqOIgBKSkrIyMggPz8f7UQcG+cc5eXllJSUMGjQoKC7IyLHWbzOAfQHiqOel/hth2ovaaP9AGY21cyKzKyorKzsgPG1tbXk5OSo+MeBmZGTk6O9KZGQaPcngZ1zTzrnCp1zhT17HnAZK4CKfxzpvRQJj3h9D2AzMCDqeZ7fthnvMFB0+wK/Pa+N6UVEQquuIcKuvXXsqN7Hzj117NxTx47qOnbu2cfU80+me1p8z83FKwBeA+4xsxfxTgLvds6Vmtlc4D/MLNuf7jLgPufcTjOrNLNxeCeBvwD8Kk59OaEqKip4/vnn+drXvnZE811++eU8//zzZGVlHXSaH/zgB1xwwQVceumlx9pNEQlAU0Evr66jfI9X1KOHveLeVOj3UVXb0OZyEhOMq0f1DyYAzOwFvC35XDMrwbuyJxnAOfdrvKt4LgfWAXuBO/1xO83s34GF/qIeaDohDHwN7+qirngnf4/4BHB7UFFRweOPP35AADQ0NJCUdPC3d/bs2Ydd9gMPPHDM/ROR+KlvjDQX8Z17vELe9nAd5dX7qDxEQc9OSyG3Wwo90lM4vV8mud260CPde+61d2kezkxNJiEh/odnY70K6ObDjHfA3QcZNxOY2UZ7ETA8ltdvz6ZPn8769esZNWoUycnJpKamkp2dzerVq1mzZg3XXHMNxcXF1NbW8o1vfIOpU6cCLbe1qK6uZtKkSZx33nm8++679O/fn1dffZWuXbsyZcoUrrzySq6//nry8/O54447eP3116mvr+ePf/wjQ4YMoaysjFtuuYUtW7Zwzjnn8Oabb7Jo0SJyc3MDfmdEjo5zjp176qje10BDxNEYcdQ3RmiMuLafNzoaIpHmcQ2Het7oqI84GqOeN0S88S3TNr1OhLoGR8Xeli309l7Qj1SHuhfQ4dz/+ko+2lIZ12UO65fJDz93+kHH/+QnP2HFihUsWbKEBQsWcMUVV7BixYrmyyhnzpxJjx49qKmp4cwzz+S6664jJydnv2WsXbuWF154gaeeeoobbriBl19+mdtuu+2A18rNzeXDDz/k8ccf56GHHuLpp5/m/vvv5+KLL+a+++7jjTfe4De/+U1c11/keHHOUb6njjXbqli7rZq126tYs62atduq2LW3/ri+dlKCkZhgJCUYSYkJzc+TExOi2o2khASy0pIZ1kEK+pHqVAHQHpx11ln7XUP/6KOP8sorrwBQXFzM2rVrDwiAQYMGMWrUKADGjh3Lxo0b21z25MmTm6f505/+BMA777zTvPyJEyeSnZ3d5rwiQYm10GekJnFq7wwmDu/DKb0yyOqa3FyEWxdlr1g3FfEDn3vT7f+8aRmJCaar3XydKgAOtaV+oqSnpzcPL1iwgL/+9a+89957pKWlMWHChDavse/SpUvzcGJiIjU1NW0uu2m6xMREGhra3hUVCdKO6n1HXOhP7d2NU3tn0CujiwrzCdapAiAIGRkZVFVVtTlu9+7dZGdnk5aWxurVq3n//ffj/vrjx49n1qxZfPe732XevHns2rUr7q8h0lpToV+3vZo127xCv257NTv31DVPo0Lf/ikAjlFOTg7jx49n+PDhdO3ald69ezePmzhxIr/+9a8ZOnQop512GuPGjYv76//whz/k5ptv5tlnn+Wcc86hT58+ZGRkxP11JJyOpNBfNqw3Bb29Ql/QK4PemSr07Z15F/B0DIWFha71D8KsWrWKoUOHBtSj4O3bt4/ExESSkpJ47733uOuuu1iyZMkxLTPs72ln1tAYoXpfA1W1DVTva/CH65ufV9U2ULJrr38Ip1Wh75JEgb8VX9A7g4Je3rAKfftnZoucc4Wt27UH0MFt2rSJG264gUgkQkpKCk899VTQXZLjoKExwp59jVTW1jcX7urahpbnUQXcK+Yt7VW1DVT5wzX1jYd9raZC37RFr0LfeSkAOriCggIWL14cdDfkKO3eW8/SkgqWFlfw6c69VEUV9KqmLfUYC7cZdOuSRGZqMt26JNEtNYmstBQG9EgjIzXJa+uS7A2nJpHhT5PhT980TVpKogp9SCgARE6Q2vpGVpVWsrS4gqUlu1laXMGGHXuax/fJTCWzq1eQs9JSyOuRRkZzYU5uLtpNBbypaDcVcBVuOVIKAJHjIBJxbNhRzZLi3X7Br2BVaSX1jd45t14ZXRg1IIvrxuYxakAWZ+R1JzNVP8IjJ5YCQCQOtlXWsqS4ornYLyveTdU+77sa6SmJjMjL4ovnDWbUgO6MHJBFn8xUba1L4BQAIkeoqrae5SW7WeIfu19avJutld4X/JISjCF9M7h6dD9G5mUxakAWg3t2I7ED3iZAOj8FwAnWrVs3qqur2bJlC9OmTeOll146YJoJEybw0EMPUVh4wFVbzR555BGmTp1KWloaENvtpeXI1TVE+HhrFUuKd3mHc0oqWF9WTdPV0/k5aZw9uAcj87IYOSCL0/tlkpqcGGynRWKkAAhIv3792iz+sXrkkUe47bbbmgMglttLy6E559hYvpelxRXe4ZySClZuqaSuIQJATnoKowZkcdXIfowckMWI/t3JTk8JuNciR08BcIymT5/OgAEDuPtu727YP/rRj0hKSmL+/Pns2rWL+vp6HnzwQa6++ur95tu4cSNXXnklK1asoKamhjvvvJOlS5cyZMiQ/e4FdNddd7Fw4UJqamq4/vrruf/++3n00UfZsmULF110Ebm5ucyfP7/59tK5ubk8/PDDzJzp3YH7S1/6Evfeey8bN2486G2nw2rPvgYWfbqLhRt3sqS4gmUlu9ld492zpmtyImf0784d5wxk5IAsRuZlkZfdVcftpVPpXAEwZzpsXR7fZfY5Ayb95KCjb7zxRu69997mAJg1axZz585l2rRpZGZmsmPHDsaNG8dVV1110OLxxBNPkJaWxqpVq1i2bBljxoxpHjdjxgx69OhBY2Mjl1xyCcuWLWPatGk8/PDDzJ8//4D7/i9atIjf/va3fPDBBzjnOPvss7nwwgvJzs6O+bbTndXeugaKNu7i/Q3lvL+hnGUlu2mIOBIMTuuTyeVn9Gk+lFPQqxtJie3+J7NFjknnCoAAjB49mu3bt7NlyxbKysrIzs6mT58+fPOb3+Stt94iISGBzZs3s23bNvr06dPmMt566y2mTZsGwIgRIxgxYkTzuFmzZvHkk0/S0NBAaWkpH3300X7jW3vnnXe49tprm+9KOnnyZN5++22uuuqqmG873VnsrfO28L2Cv5OlxRU0RByJCcaIvO58+YLBjBucQ+HAbNK76KMg4dO5/tcfYkv9ePr85z/PSy+9xNatW7nxxht57rnnKCsrY9GiRSQnJ5Ofn9/mbaAP55NPPuGhhx5i4cKFZGdnM2XKlKNaTpNYbzvdUangixwZfQri4MYbb+TLX/4yO3bs4B//+AezZs2iV69eJCcnM3/+fD799NNDzn/BBRfw/PPPc/HFF7NixQqWLVsGQGVlJenp6XTv3p1t27YxZ84cJkyYALTchrr1IaDzzz+fKVOmMH36dJxzvPLKKzz77LPHZb2DFmvBHzswm24q+CIH0KciDk4//XSqqqro378/ffv25dZbb+Vzn/scZ5xxBoWFhQwZMuSQ8991113ceeedDB06lKFDhzJ27FgARo4cyejRoxkyZAgDBgxg/PjxzfNMnTqViRMn0q9fP+bPn9/cPmbMGKZMmcJZZ50FeCeBR48e3SkO9+yta+DDTyuaj+EvLamgvlEFX+Ro6XbQcoD28p7W1DVGbeHvX/DP6N+dcYNzGDe4B4X5PVTwRQ5Bt4OWdu9wBf+L5w1WwReJI32KJDA1dY18uKml4C8pVsEXOZE6xafKOacv6MTJ8T4kuHtvPW+u2sac5aW8vXYHdY0RFXyRgHT4T1lqairl5eXk5OQoBI6Rc47y8nJSU1Pjutyde+p486OtzF6+lX+u20FDxNE/qyu3jRvI+afmUjgwmwzdClnkhOvwAZCXl0dJSQllZWVBd6VTSE1NJS8v75iXU1a1j3kfbWXO8q28t6GcxojjpB5pfPH8QVw+vC8j8rorsEUC1uEDIDk5mUGDBgXdDcG7J/7clVuZvbyU//1kJxEHg3LT+eqFg5k0vC+n98tU0RdpRzp8AEiwtlTU8MaKrcxZUUrRp7twDgp6deOeiwu4/Iw+nNY7Q0VfpJ1SAMgRK965lzdWbGX2ilIWb6oAYEifDL556alMGt6Hgt4ZAfdQRGKhAJCYbNyxhzn+lv6ykt0ADO+fyXc+exqThvdhcM9uAfdQRI6UAkAOan1ZNXOWlzJ7+VY+Kq0EYOSALO6bNIRJw/tyUk5awD0UkWOhAJBmzjnWbq9m9vJS5izfysfbqgAYOzCb718xlInD+5CXraIv0lkoAELOOceq0irmrChl9vJS1pftwQzOzO/Bjz43jInD+9Kne3y/FyAi7YMCIIScc6zYXMnsFaXMWV7KxvK9JBiMG5zDlPGD+OzpvemVoaIv0tkpAELEOcefl5Xy8Jtr+GTHHhITjHNPzuErF57MZcN6k9Oty+EXIiKdhgIgJBZ9uosH//IRizdVMKRPBj+7fgSfGdqb7PSUoLsmIgGJKQDMbCLwSyAReNo595NW4wcCM4GewE7gNudciT/up8AV/qT/7pz7g9/+O+BCYLc/bopzbskxrY0coHjnXn76xmr+vKyUnhld+Nl1I7hubB6JCfpylkjYHTYAzCwReAz4DFACLDSz15xzH0VN9hDwjHPuv83sYuDHwO1mdgUwBhgFdAEWmNkc51ylP993nHMvxXF9xFdZW8/j89cz85+fkGAw7ZICvnLBYP0Wrog0i6UanAWsc85tADCzF4GrgegAGAZ8yx+eD/xPVPtbzrkGoMHMlgETgVlx6Lu0oaExwosLi/nFm2so31PH5DH9+c5nT6Nv965Bd01E2pmEGKbpDxRHPS/x26ItBSb7w9cCGWaW47dPNLM0M8sFLgIGRM03w8yWmdkvzKzNM5BmNtXMisysSHf8PDjnHPM/3s6kX77N9/9nBaf06sbr95zHwzeMUvEXkTbF63jAt4H/NLMpwFvAZqDROTfPzM4E3gXKgPeARn+e+4CtQArwJPBd4IHWC3bOPemPp7CwsOP8gPEJtHprJTP+soq31+4gPyeN/7p9LJcN662bsInIIcUSAJvZf6s9z29r5pzbgr8HYGbdgOuccxX+uBnADH/c88Aav73Un32fmf0WL0TkCGyvquUXb67hDwuLyUhN5t+uHMbt4waSkhTLjp2IhF0sAbAQKDCzQXiF/ybglugJ/MM7O51zEbwt+5l+eyKQ5ZwrN7MRwAhgnj+ur3Ou1LzN1GuAFXFap06vtr6R37zzCY/PX8e+hghTzh3EtEtOIStNl3SKSOwOGwDOuQYzuweYi3cZ6Ezn3EozewAocs69BkwAfmxmDu8Q0N3+7MnA2/6hiEq8y0Mb/HHPmVlPwIAlwFfjt1qdUyTieH3ZFn46ZzVbdtdy2bDeTJ80RHfiFJGjYsf7R8DjqbCw0BUVFQXdjUAs3LiTB//8EUtLdjO8fybfv2IY4wbnBN0tEekAzGyRc66wdbsuCm/nNpXv5SdvrGL28q30zuzCzz8/kmtH9ydBX+QSkWOkAGindtfU89j8dfzunxtJTDC+eempfPmCQaSl6J9MROJD1aSdqW+M8PwHm3jkr2uoqKnn82Pz+JfLTqN3pu7OKSLxpQBoJ5xz/H31dmbMXsWGsj2ce3IO37tiKKf36x5010Skk1IAtAMrt+xmxl9W8e76cgb3TOfpLxRyydBe+iKXiBxXCoAAba+s5aF5H/PHRSVkdU3m/qtO55azTyI5UV/kEpHjTwEQgJq6Rp56ewO//sd66hsjfOm8QdxzUQHd05KD7pqIhIgC4ASKRByvLN7M/5v7MVsra7n8jD58d+IQBuakB901EQkhBcAJUlvfyJTf/i/vb9jJyLzu/OqW0ZyZ3yPobolIiCkATgDnHD94dQXvb9jJjGuHc/OZJ+mLXCISOAXACfDcB5uYVVTCtItP4dazBwbdHRERILYfhJFjULRxJ/e/vpKLTuvJvZeeGnR3RESaKQCOo22Vtdz13If0z+rKIzeN1mEfEWlXdAjoOKlriHDX7xexZ18Dv//i2XTvqks8RaR9UQAcJ/e/vpIPN1Xw2C1jOK1PRtDdERE5gA4BHQd/WLiJ5z7YxFcvPJkrRvQNujsiIm1SAMTZkuIK/u1/VnJ+QS7f+expQXdHROSgFABxVFa1j68+u4hemV149KbRJOqkr4i0YzoHECf1jRHufu5DKmrqePmuc8lO1w+0i0j7pgCIkxl/WcX/btzJL28apXv4i0iHoENAcfDyohJ+9+5GvnjeIK4e1T/o7oiIxEQBcIyWl+zmX19ZzrjBPbhv0pCguyMiEjMFwDEor97HV3+/iJz0FB67ZQxJ+iEXEelAdA7gKDU0Rvj6C4spq97Hy189l5xuXYLukojIEdEm61H66RureXd9OTOuGc4ZeTrpKyIdjwLgKLy6ZDNPvf0JXzhnIJ8vHBB0d0REjooC4Ah9tKWS7768jDPzs/n+FcOC7o6IyFFTAByBir11fOX3RXTvmsxjt44hJUlvn4h0XDoJHKPGiOPrLyxm6+5a/vCVc+iVkRp0l0REjokCIEY/n/cxb6/dwY8nn8GYk7KD7o6IyDHTMYwYzF5eyuML1nPzWSdx81knBd0dEZG4UAAcxpptVXz7j0sZfVIWP7pKJ31FpPNQABzC7pp6pj5TRFpKEk/cOpYuSYlBd0lEJG4UAAcRiTi++YcllOyq4YnbxtCnu076ikjnogA4iEf+tpa/r97ODz43jDPzewTdHRGRuFMAtGHeyq08+re1XD82j9vHDQy6OyIix0VMAWBmE83sYzNbZ2bT2xg/0Mz+ZmbLzGyBmeVFjfupma3wHzdGtQ8ysw/8Zf7BzNrFT2it217Nt2YtZURedx68Zjhm+llHEemcDhsAZpYIPAZMAoYBN5tZ68thHgKecc6NAB4AfuzPewUwBhgFnA1828wy/Xl+CvzCOXcKsAv44rGvzrGpqq3nK88WkZKUwBO3jSU1WSd9RaTzimUP4CxgnXNug3OuDngRuLrVNMOAv/vD86PGDwPecs41OOf2AMuAieZtVl8MvORP99/ANUe/GscuEnH8y6ylbCzfy3/eMpr+WV2D7I6IyHEXSwD0B4qjnpf4bdGWApP94WuBDDPL8dsnmlmameUCFwEDgBygwjnXcIhlAmBmU82syMyKysrKYlmno/L4gnXM+2gb/3r5UM49Ofe4vY6ISHsRr5PA3wYuNLPFwIXAZqDROTcPmA28C7wAvAc0HsmCnXNPOucKnXOFPXv2jFN39zd/9XZ+/uYarh7Vj/8zPv+4vIaISHsTSwBsxttqb5LntzVzzm1xzk12zo0Gvue3Vfh/ZzjnRjnnPgMYsAYoB7LMLOlgyzxRNu7Yw7QXFzOkTyY/mTxCJ31FJDRiCYCFQIF/1U4KcBPwWvQEZpZrZk3Lug+Y6bcn+oeCMLMRwAhgnnPO4Z0ruN6f5w7g1WNdmSO1Z18DX3l2EYkJxpO3j6Vrik76ikh4HDYA/OP09wBzgVXALOfcSjN7wMyu8iebAHxsZmuA3sAMvz0ZeNvMPgKeBG6LOu7/XeBbZrYO75zAb+K0TjFxzvF/X1rG2u1V/Orm0QzokXYiX15EJHAx3Q7aOTcb71h+dNsPooZfouWKnuhpavGuBGprmRvwrjAKxH+9tYG/LC9l+qQhnF9wfM4tiIi0Z6H8JvDba8v42RurueKMvnzlgsFBd0dEJBChC4DinXv5+guLOaVXN352vU76ikh4hSoAauoamfrsIiIRx5O3F5LeRT+IJiLhFZoK6Jxj+p+WsXprJTPvOJP83PSguyQiEqjQ7AHM/OdGXl2yhW9deioXDekVdHdERAIXigB4d/0O/mP2Ki4b1pu7Lzol6O6IiLQLnT4AnHM88te15Oek8fMbRpKQoJO+IiIQgnMAZsZv7iikYm89GanJQXdHRKTd6PQBAJCRmqziLyLSSqc/BCQiIm1TAIiIhJQCQEQkpBQAIiIhpQAQEQkpBYCISEgpAEREQkoBICISUgoAEZGQUgCIiISUAkBEJKQUACIiIaUAEBEJKQWAiEhIKQBEREJKASAiElIKABGRkFIAiIiElAJARCSkFAAiIiGlABARCSkFgIhISCkARERCSgEgIhJSCgARkZBSAIiIhJQCQEQkpGIKADObaGYfm9k6M5vexviBZvY3M1tmZgvMLC9q3M/MbKWZrTKzR83M/PYF/jKX+I9e8VstERE5nMMGgJklAo8Bk4BhwM1mNqzVZA8BzzjnRgAPAD/25z0XGA+MAIYDZwIXRs13q3NulP/YfqwrIyIisYtlD+AsYJ1zboNzrg54Ebi61TTDgL/7w/OjxjsgFUgBugDJwLZj7bSIiBy7WAKgP1Ac9bzEb4u2FJjsD18LZJhZjnPuPbxAKPUfc51zq6Lm+61/+Offmg4NtWZmU82syMyKysrKYuiuiIjEIl4ngb8NXGhmi/EO8WwGGs3sFGAokIcXGheb2fn+PLc6584Azvcft7e1YOfck865QudcYc+ePePUXRERiSUANgMDop7n+W3NnHNbnHOTnXOjge/5bRV4ewPvO+eqnXPVwBzgHH/8Zv9vFfA83qEmERE5QWIJgIVAgZkNMrMU4CbgtegJzCzXzJqWdR8w0x/ehLdnkGRmyXh7B6v857n+vMnAlcCKY18dERGJ1WEDwDnXANwDzAVWAbOccyvN7AEzu8qfbALwsZmtAXoDM/z2l4D1wHK88wRLnXOv450Qnmtmy4AleHsUT8VtrURE5LDMORd0H2JWWFjoioqKgu6GiEiHYmaLnHOFrdv1TWARkZBSAIiIhJQCQEQkpBQAIiIhpQAQEQkpBYCISEgpAEREQkoBICISUgoAEZGQUgCIiISUAkBEJKQUACIiIaUAEBEJKQWAiEhIKQBEREJKASAiElIKABGRkFIAiIiElAJARCSkFAAiIiGlABARCSkFgIhISCkARERCSgEgIhJSCgARkZBSAIiIhJQCQEQkpBQAIiIhpQAQEQkpBYCISEgpAEREQkoBICISUgoAEZGQUgCIiISUAkBEJKQUACIiIRVTAJjZRDP72MzWmdn0NsYPNLO/mdkyM1tgZnlR435mZivNbJWZPWpm5rePNbPl/jKb20VE5MQ4bACYWSLwGDAJGAbcbGbDWk32EPCMc24E8ADwY3/ec4HxwAhgOHAmcKE/zxPAl4EC/zHxWFdGRERiF8sewFnAOufcBudcHfAicHWraYYBf/eH50eNd0AqkAJ0AZKBbWbWF8h0zr3vnHPAM8A1x7QmIiJyRGIJgP5AcdTzEr8t2lJgsj98LZBhZjnOuffwAqHUf8x1zq3y5y85zDJFROQ4itdJ4G8DF5rZYrxDPJuBRjM7BRgK5OEV+IvN7PwjWbCZTTWzIjMrKisri1N3RUQklgDYDAyIep7ntzVzzm1xzk12zo0Gvue3VeDtDbzvnKt2zlUDc4Bz/PnzDrXMqGU/6ZwrdM4V9uzZM8bVEhGRw4klABYCBWY2yMxSgJuA16InMLNcM2ta1n3ATH94E96eQZKZJePtHaxyzpUClWY2zr/65wvAq3FYHxERidFhA8A51wDcA8wFVgGznHMrzewBM7vKn2wC8LGZrQF6AzP89peA9cByvPMES51zr/vjvgY8Dazzp5kTlzUSEZGYmHcRTsdQWFjoioqKgu6GiEiHYmaLnHOFrdv1TWARkZBSAIiIhJQCQEQkpBQAIiIhlRR0B+QE2lcNn/wDNv4TcgZDwWWQdVLQvRKRgCgAOjPnoHw9rJ0Ha+fCp+9CYx0kJEGkwZum5xAo+IwXBgPGQVJKsH0WkRNGAdDZ1Nd4W/hr53mPXZ947bmnwVlTvUJ/0jlQ8ak3fs1ceP/X8O6vICUDTp7gTXPKZyCzb6CrIiLHlwKgM9i1Eda+6T0+eQsaaiCpKwy6AM6529vCz87ff57cAu9xzt2wrwo2/MMLhHV/hVX+d/X6nOGFQcFl0L8QEvXfReLAOdhXCTUVUFtxkL+7oWu29/82Ox96DIKMvpCQGHTvOxV9EawjaqiDTe/5W/lvwo6PvfbsfCj4rFew88dDctcjX7ZzsG1ly7KLPwDXCKlZcMol/t7BpZCeG9dVCkxjg1eMumaDfpMods55Gw5NBbtm1yGKeRvF3TUefNkJSdAl88DpElO8c1ZNoZCdD9mD/L8DoUvG8V3nDuxgXwRTAHQUlVv8rfx5sGEB1FV7H4iB41u20nNOjn8Rq9kF6+d7r73uTdhTBhj0H+u/7meg7yhIaOcXlNVWQvla2LEOdqzxH2th53rvvEhiCmT2g8w872/3/pDZ32/zh9NzO2dI7AcnR/EAAAb4SURBVKv2/l33lEH1dqjZGVWwdx19EU/Ngq5ZMf7NbhlOSffe58YG2F3s7eEe8PjE60O0tNyWvYX9QiIfMvq1//+jx5ECoKNpbICShS1b4tuWe+2ZeS0nbQddAF26nbg+RSJQuqQliDYvAhyk9/TOGRR8Bk6+yPswByESgcrNXnEvb1Xoq0pbprNE6DG45TBYt95QvQ12b/aCttL/G6nff/kdJSSc8wp1dRns2e4V9T07ooajiv2eMqjf2/ZyjriI+4U8uogfTzW7DgyGnZ94f3eXtLH3MPDAYOgxyGs/kZ+jACgAOoLqMu8Y/Np5sP5v3haOJXonbZuKfq+hwReYJnt2wLq/tZw7qK3w+jvg7Jb+9j49/v2tr/EL/Fr/saal6EcXsy7d/SJ/atTfU70P/uGudopEYO8Or5A0h8LmqJAogcrSNkKii3fy/ICQiAqKowmJSCPsLfeLtl/Qm4ar/YIePdy6XwCW4G0ld+vlhXZ6z5bhbr0gvZfXt7Qcr5indGs//9eOVGO9928XvcfQHBIbYV+rvYf0nm0fWsoa4L1vjfX+o857b5uGG+u8jbWm4UjUcPQ8jfX+fG20H7DMVstvWuZtLx94Li9G4Q6Aje/A3p2QkgbJ6d7WSUo6JKe1DAdxcikSgS2LW67Y2bIYb4u6V8vhlcETvA9je9fY4O0RNK3L1mVee0a/ljAYfGHsx2md8wpc+dqWrfimQl9RjPdrowDmfUibint0oU/veXwLWCTiFdumcKjcEhUYRxESyV3b3kKv3u4Vf9r4rCYkt13Em4a79fTbekJaD51EbVKzq2VvofWj9d5D3BgkdfH+zRKTvb2SxBTv4orEFK8tIbllODF6OAUue9D7/3I0rxzqAPj9dd4W6qEkpe4fCLEMHzAu3QuZ5uF07x88ugjt3Qnr/+4fU/+rt5WJQd6ZLUW/z4iOf7yysjRqb2Y+1FV5/7kHnttyziK3wNu62fnJ/odrmv5Gb6Ulp0HOKQcW+pyTj+5k94lyyJBoamsVEsnpBy/iTcNNRT+1e8fdSm+vGutbzj1U+L+GG12Io4vzoQp2Qqu2AMM33AGwe7N3Yqtur3fytH7vIYb3eI82h/3pjmTrwBJawiA5FSo2gYtA1x7e1TQFl8HJF0N6zpGvV0fRUOddTdR0PqNsldeeluttiUW/nxl999+KbxruzCfxmkKifq9X2FPSg+6RdDLhDoB4cs47HnfQoKj2guJgwz0G+dfVjw3v7njFJi8ISoq8XdrcUyH3FMgpgNTMoHsn0ukcLAD0zZ4jZf5xvKQu3jFVOXJZJ8GZX/QeIhKYTrpPLSIih6MAEBEJKQWAiEhIKQBEREJKASAiElIKABGRkFIAiIiElAJARCSkOtQ3gc2sDPj0KGfPBXbEsTsdnd6PFnov9qf3Y3+d4f0Y6Jzr2bqxQwXAsTCzora+Ch1Wej9a6L3Yn96P/XXm90OHgEREQkoBICISUmEKgCeD7kA7o/ejhd6L/en92F+nfT9Ccw5ARET2F6Y9ABERiaIAEBEJqVAEgJlNNLOPzWydmU0Puj9BMbMBZjbfzD4ys5Vm9o2g+9QemFmimS02sz8H3ZegmVmWmb1kZqvNbJWZnRN0n4JiZt/0PycrzOwFM0sNuk/x1ukDwMwSgceAScAw4GYzGxZsrwLTAPyLc24YMA64O8TvRbRvAKuC7kQ78UvgDefcEGAkIX1fzKw/MA0odM4NBxKBm4LtVfx1+gAAzgLWOec2OOfqgBeBqwPuUyCcc6XOuQ/94Sq8D3f/YHsVLDPLA64Ang66L0Ezs+7ABcBvAJxzdc65imB7FagkoKuZJQFpwJaA+xN3YQiA/kBx1PMSQl70AMwsHxgNfBBsTwL3CPB/gUjQHWkHBgFlwG/9Q2JPm1l60J0KgnNuM/AQsAkoBXY75+YF26v4C0MASCtm1g14GbjXOVcZdH+CYmZXAtudc4uC7ks7kQSMAZ5wzo0G9gChPGdmZtl4RwoGAf2AdDO7LdhexV8YAmAzMCDqeZ7fFkpmloxX/J9zzv0p6P4EbDxwlZltxDs0eLGZ/T7YLgWqBChxzjXtFb6EFwhhdCnwiXOuzDlXD/wJODfgPsVdGAJgIVBgZoPMLAXvRM5rAfcpEGZmeMd3VznnHg66P0Fzzt3nnMtzzuXj/b/4u3Ou023lxco5txUoNrPT/KZLgI8C7FKQNgHjzCzN/9xcQic8IZ4UdAeON+dcg5ndA8zFO5M/0zm3MuBuBWU8cDuw3MyW+G3/6pybHWCfpH35OvCcv7G0Abgz4P4Ewjn3gZm9BHyId/XcYjrhLSF0KwgRkZAKwyEgERFpgwJARCSkFAAiIiGlABARCSkFgIhISCkARERCSgEgIhJS/x+VCTCcOyPL5gAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"code","metadata":{"id":"KuLWQ3IB6E4i","executionInfo":{"status":"ok","timestamp":1604773138342,"user_tz":420,"elapsed":3418,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f4c8be9e-960b-416f-a757-56c1063a9389","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":16,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 3s 9ms/step - loss: 0.0973 - accuracy: 0.9793\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Jx3kiSqK6E4l","executionInfo":{"status":"ok","timestamp":1604773139301,"user_tz":420,"elapsed":509,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"886fc4e3-a644-423d-a159-61784b7f9d16","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":17,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9793000221252441\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"1j7a2ETC6E4n"},"source":["## 1. Deep CNN - 1\n","- Deeper network with 6 convolutional layers and 3 pooling layers\n","- Note that two (3x3) convolutional layers are stacked\n","    - It is equivalent to using one (5X5) convolutional layer, but with fewer parameters\n","    - Similar to the structured used in VGGnet"]},{"cell_type":"code","metadata":{"id":"TCuB2iBT6E4n","executionInfo":{"status":"ok","timestamp":1604773143562,"user_tz":420,"elapsed":549,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def deep_cnn():\n","    model = Sequential()\n","    \n","    model.add(Conv2D(input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]), filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same'))\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    \n","    # prior layer should be flattend to be connected to dense layers\n","    model.add(Flatten())\n","    # dense layer with 50 neurons\n","    model.add(Dense(50, activation = 'relu'))\n","    # final layer with 10 neurons to classify the instances\n","    model.add(Dense(10, activation = 'softmax'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model    "],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"qwQsZPPR6E4s","executionInfo":{"status":"ok","timestamp":1604773145075,"user_tz":420,"elapsed":492,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["model = deep_cnn()"],"execution_count":19,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"MPMNt-IM6E4u","executionInfo":{"status":"ok","timestamp":1604773145357,"user_tz":420,"elapsed":404,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"7cb5f372-7aea-4c6d-ef99-b6bffcd30094","colab":{"base_uri":"https://localhost:8080/"}},"source":["model.summary()"],"execution_count":20,"outputs":[{"output_type":"stream","text":["Model: \"sequential_1\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","conv2d_1 (Conv2D)            (None, 28, 28, 50)        500       \n","_________________________________________________________________\n","activation_1 (Activation)    (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","conv2d_2 (Conv2D)            (None, 28, 28, 50)        22550     \n","_________________________________________________________________\n","activation_2 (Activation)    (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","max_pooling2d_1 (MaxPooling2 (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","conv2d_3 (Conv2D)            (None, 14, 14, 50)        22550     \n","_________________________________________________________________\n","activation_3 (Activation)    (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","conv2d_4 (Conv2D)            (None, 14, 14, 50)        22550     \n","_________________________________________________________________\n","activation_4 (Activation)    (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","max_pooling2d_2 (MaxPooling2 (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","conv2d_5 (Conv2D)            (None, 7, 7, 50)          22550     \n","_________________________________________________________________\n","activation_5 (Activation)    (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","conv2d_6 (Conv2D)            (None, 7, 7, 50)          22550     \n","_________________________________________________________________\n","activation_6 (Activation)    (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","max_pooling2d_3 (MaxPooling2 (None, 3, 3, 50)          0         \n","_________________________________________________________________\n","flatten_1 (Flatten)          (None, 450)               0         \n","_________________________________________________________________\n","dense_2 (Dense)              (None, 50)                22550     \n","_________________________________________________________________\n","dense_3 (Dense)              (None, 10)                510       \n","=================================================================\n","Total params: 136,310\n","Trainable params: 136,310\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"OJa65OgV6E4x","executionInfo":{"status":"ok","timestamp":1604776093332,"user_tz":420,"elapsed":2940517,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"3a794b74-9a9a-4807-ef01-f3490ed96ed7","colab":{"base_uri":"https://localhost:8080/"}},"source":["%%time\n","history = model.fit(X_train, y_train, batch_size = 50, validation_split = 0.2, epochs = 10, verbose = 1)"],"execution_count":21,"outputs":[{"output_type":"stream","text":["Epoch 1/10\n","960/960 [==============================] - 300s 313ms/step - loss: 0.1896 - accuracy: 0.9412 - val_loss: 0.0490 - val_accuracy: 0.9854\n","Epoch 2/10\n","960/960 [==============================] - 293s 305ms/step - loss: 0.0519 - accuracy: 0.9835 - val_loss: 0.0494 - val_accuracy: 0.9852\n","Epoch 3/10\n","960/960 [==============================] - 292s 304ms/step - loss: 0.0432 - accuracy: 0.9870 - val_loss: 0.0491 - val_accuracy: 0.9858\n","Epoch 4/10\n","960/960 [==============================] - 293s 305ms/step - loss: 0.0364 - accuracy: 0.9889 - val_loss: 0.0451 - val_accuracy: 0.9863\n","Epoch 5/10\n","960/960 [==============================] - 292s 305ms/step - loss: 0.0306 - accuracy: 0.9907 - val_loss: 0.0530 - val_accuracy: 0.9837\n","Epoch 6/10\n","960/960 [==============================] - 293s 305ms/step - loss: 0.0271 - accuracy: 0.9916 - val_loss: 0.0353 - val_accuracy: 0.9893\n","Epoch 7/10\n","960/960 [==============================] - 293s 305ms/step - loss: 0.0253 - accuracy: 0.9919 - val_loss: 0.0399 - val_accuracy: 0.9887\n","Epoch 8/10\n","960/960 [==============================] - 293s 305ms/step - loss: 0.0237 - accuracy: 0.9927 - val_loss: 0.0526 - val_accuracy: 0.9852\n","Epoch 9/10\n","960/960 [==============================] - 294s 306ms/step - loss: 0.0222 - accuracy: 0.9927 - val_loss: 0.0379 - val_accuracy: 0.9907\n","Epoch 10/10\n","960/960 [==============================] - 294s 306ms/step - loss: 0.0173 - accuracy: 0.9948 - val_loss: 0.0541 - val_accuracy: 0.9855\n","CPU times: user 1h 28min 49s, sys: 2min 14s, total: 1h 31min 4s\n","Wall time: 49min\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"oJCOwXi_6E4z","executionInfo":{"status":"ok","timestamp":1604776094477,"user_tz":420,"elapsed":1117,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"b02b9252-0143-48a9-ce01-41425f934201","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.legend(['training', 'validation'], loc = 'upper left')\n","plt.show()"],"execution_count":22,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXRT95338ffX8oZ3Yxu8siQhgAOODc5WmkBIOiVLyb616ZScp2VO2k7azmSeIdN5kgltJp2nTE8mp9uTNPRM2qQdSptpMkNKNkjSJumAI0FYAiEEJGMWA5YXvEr6PX9c2QhjsIxlX+nq+zqHI+neK+lrAR/9/L33/q4YY1BKKeVcKXYXoJRSamxp0CullMNp0CullMNp0CullMNp0CullMOl2l3AYMXFxWbatGl2l6GUUgmloaHhqDGmZKh1cRf006ZNY/PmzXaXoZRSCUVE9p9pnbZulFLK4TTolVLK4TTolVLK4eKuRz+Uvr4+Ghsb6e7utrsUx8jMzKSyspK0tDS7S1FKjbGECPrGxkZyc3OZNm0aImJ3OQnPGMOxY8dobGxk+vTpdpejlBpjCdG66e7upqioSEM+RkSEoqIi/Q1JqSSREEEPaMjHmH6eSiWPhGjdKKWUUwVDht2H22nY30KKCJ+/bErM30ODPkp+v5/nn3+er371qyN63vXXX8/zzz9PQUHBGbd5+OGHueqqq7j22mtHW6ZSKs61d/fh8flp2N9Cw/4WPF4/7T0BAOZNKdCgt5Pf7+fHP/7xaUEfCARITT3zx7hu3bphX3vlypWjrk8pFX+MMfiOd9HgPR4Odj+7DrURMiACs0rzuKmunPlTC5k/ZSJVEyeMSR0a9FFasWIFH3/8MbW1taSlpZGZmUlhYSEffvghu3fv5uabb8bn89Hd3c03vvENli9fDpyc0qGjo4PrrruOT3/607zzzjtUVFTw+9//ngkTJrBs2TJuvPFGbr/9dqZNm8aXvvQlXnrpJfr6+vjNb37DrFmzaG5u5vOf/zxNTU1cccUVvPrqqzQ0NFBcXGzzJ6OU6tcTCLLtQBvvh0frDd4Wmtt7AMjJSKVuSgGfvWYG86cWUltVQG7m+BzenHBB/+hL29nR1BbT16wuz+ORz1101m2+973vsW3bNjweDxs3buSGG25g27ZtA4cnrl69mokTJ9LV1cUll1zCbbfdRlFR0Smv8dFHH/GrX/2Kp59+mjvvvJPf/va33Hvvvae9V3FxMe+//z4//vGPWbVqFT/72c949NFHWbx4MQ899BB/+MMfeOaZZ2L3ASilzklzew/ve1t4f38Lm/e38EFjK73BEABTi7K48oJi5k0tZP7UQi6cnIsrxZ6DIBIu6OPFpZdeesox6E8++SQvvPACAD6fj48++ui0oJ8+fTq1tbUAzJ8/n3379g352rfeeuvANr/73e8A+OMf/zjw+kuWLKGwsDCmP49S6uyCIcNHR6ydpg37rNH6/mOdAKS7Uphbmc+yBdOYN8UK9pLcDJsrPinhgn64kfd4yc7OHri/ceNGXnvtNd59912ysrJYtGjRkMeoZ2Sc/It3uVx0dXUN+dr927lcLgKBQIwrV0pF42w7TYtzMpg/tYAvXDaF+VMnMqcij4xUl80Vn1nCBb1dcnNzaW9vH3Jda2srhYWFZGVl8eGHH/Lee+/F/P0XLFjAmjVr+Pu//3teeeUVWlpaYv4eSiWr4Xaazpycy9LacuqnndxpmkjnomjQR6moqIgFCxYwZ84cJkyYwOTJkwfWLVmyhJ/+9KfMnj2bmTNncvnll8f8/R955BHuuecefvGLX3DFFVdQWlpKbm5uzN9HqVgIhQytXX0cO9HDsY5ejp0I/+no4fiJ3vCyHlq7Ahhj7C6Xox29HO2wf6fpWJF4+JAj1dfXm8EXHtm5cyezZ8+2qaL40NPTg8vlIjU1lXfffZf7778fj8czqtfUz1VFyxhDW1fACu6IoD4+RIgf7eilpbOXYGjobMmfkEZRTjpF2enkT0jHpv2Tp8jNTKNuSoHtO01HQ0QajDH1Q63TEX2C8Hq93HnnnYRCIdLT03n66aftLkklMGMM7T2BcFCfHHVbQR056rYCvKWzl77g0MGdm5lKUXY6RTkZVE3Mom5KAUXZGUzMTg8HesZAsBdmp5PmSpiZVxxDgz5BzJgxA7fbbXcZKsb6giF6AiF6+oLWbSBETyBIT1+I7iGW9QQil4dv+6z73X0Rywa/ZuRz+kJ09QUJnGHEnZ3uoijHCueKgkxqKvKZGA7q/uCemJ1OcU4Ghdlpcb0TUlk06JWKoZ5AkCNtPRxq6+ZgazeHW7s51NbNofDt4bZuTvQEBgL4TO2NaKW7UshITSEjLYWMVBcZqSmkp6aQkeYiMzWFvAlp1vrU8Po0635mmovCrDQrtHPSKQ7fFmWnk5mmwe00GvRKRcEYQ1t3gMP9od0f4BGPD7d1c+xE72nPnZDmoiw/k8l5mVwybSK5makngzccuv0B3L+sP7gzIwI8MqgzUl2kp6YkZC9ZjT8NepX0giHD0Y6ek+HdP/puDY/Kw4He2Rs87bkTs9MpzcukND+Ti6sKKMvPHHhcGg73vMzUhDoUTzmPBr1yvK7eIB8easN7vJNDg8L7UGs3R9p7TmuhpKYIk/MymZyXweyyPBbNnERpfgal+RMozcukLD+TSXkZ2p92gu5WOHEUis63u5IxE1XQi8gS4N8AF/AzY8z3Bq2fCqwGSoDjwL3GmMbwun8Bbghv+h1jzH/EqPa4lpOTQ0dHB01NTTzwwAOsXbv2tG0WLVrEqlWrqK8f8ogoAJ544gmWL19OVlYWEN20x8nsaEcPO5ra2HGwje1NbexoauWToyeIzPHsdNfAiPuK84sGRuGT8zIpy5/A5PwMirMzSNG2iLOFQuB5Dl57BDqPweyl8JlHYeJ5dlcWc8MGvYi4gB8BnwEagU0i8qIxZkfEZquAZ40x/y4ii4HHgS+KyA3APKAWyAA2isjLxpjYzkoWx8rLy4cM+Wg98cQT3HvvvQNBH820x8kgFDJ4j3eGA711INwPt/UMbFNRMIHZZXncWFNOdXke5xVnU5qfmfAnv6gYaPLAugehcRNUXQ7zl8F7P4Hdf4DL/gqufBAmOGcwFc2I/lJgjzFmL4CI/Bq4CYgM+mrgb8L3NwD/GbH8LWNMAAiIyFZgCbAmBrWPqxUrVlBVVcXXvvY1AP7pn/6J1NRUNmzYQEtLC319fXz3u9/lpptuOuV5+/bt48Ybb2Tbtm10dXVx3333sWXLFmbNmnXKXDf3338/mzZtoquri9tvv51HH32UJ598kqamJq6++mqKi4vZsGHDwLTHxcXF/OAHP2D16tUAfPnLX+ab3/wm+/btO+N0yImqJxBk96EOdhy0An17Uxs7D7ZxItwzd6UIMyblsOD8YqrL86w/ZXkUZKXbXLmKO10t8MZ3YdMzkF0MN/8Eau6GlBS45CvWund+CO7n4Op/gPn3gSvxO9zR/AQVgC/icSNw2aBttgC3YrV3bgFyRaQovPwREflXIAu4mlO/IAAQkeXAcoApU4a5usrLK+DQB1GUPQKlc+G67511k7vuuotvfvObA0G/Zs0a1q9fzwMPPEBeXh5Hjx7l8ssvZ+nSpWfc8faTn/yErKwsdu7cydatW5k3b97Auscee4yJEycSDAa55ppr2Lp1Kw888AA/+MEP2LBhw2nzzjc0NPDzn/+cP//5zxhjuOyyy1i4cCGFhYVRT4ccj/ydvew42GaN0MOj9D1HOgaO+c5OdzG7LI/b5ldyUXke1WX5zJico4cEqrOLbNN0tcCly60gjxy155XBzT+Cy5bD+m9bI/7/eRr+4rsw4zPWpDcJKlZfVQ8CPxSRZcBbwAEgaIx5RUQuAd4BmoF3gdMOXTDGPAU8BdYUCDGqKabq6uo4cuQITU1NNDc3U1hYSGlpKd/61rd46623SElJ4cCBAxw+fJjS0tIhX+Ott97igQceAKCmpoaampqBdWvWrOGpp54iEAhw8OBBduzYccr6wf74xz9yyy23DMyieeutt/L222+zdOnSqKdDtpMxhgP+roERen+4H/Cf/C1ncl4G1WV5XDN7EtVl+VxUnseUiVmJ3TsPhmcjdcAoMWEMbtNc/30oO/P/Lcouhi+9BLvWwSv/CM/fAeddDZ99DCbHx+y5IxXNv7YDQFXE48rwsgHGmCasET0ikgPcZozxh9c9BjwWXvc8sHtUFQ8z8h5Ld9xxB2vXruXQoUPcddddPPfcczQ3N9PQ0EBaWhrTpk0bcnri4XzyySesWrWKTZs2UVhYyLJly87pdfpFOx3yeOkLhthzpGNghN5/29rVB1gDpfOKs5k3tZAvXjGV6rI8ZpflxdV83qMWDMDmZ+CNx6CnFdJzIDMfMgvCt/nW6LL//tmWp+dYrQZ1dv1tms2rIavo1DbNcERg1g1wwWdg08/gze/BTz8N8/4Srv425Ewa+/pjKJqg3wTMEJHpWAF/N/D5yA1EpBg4bowJAQ9hHYHTvyO3wBhzTERqgBrglRjWP67uuusuvvKVr3D06FHefPNN1qxZw6RJk0hLS2PDhg3s37//rM+/6qqreP7551m8eDHbtm1j69atALS1tZGdnU1+fj6HDx/m5ZdfZtGiRcDJ6ZEHt26uvPJKli1bxooVKzDG8MILL/CLX/xiTH7ukTDGsP9YJx6fH7e3BY/Pz86D7QNX3clMS2FWaR431JRRXZbHReV5zCzNJSvdwSPc/e9aI8rD26yR4ZQrrEP6uluh22/dtjXC4e3W/Z7Ws7+epEBG3lm+HArO/qWRmpnQbYhhhUKw5Xl49WEr7C/5yultmmilpsMVX4WL74Y3/8UK/Q9+C1f+DVz+VUjLjH39Y2DY/13GmICIfB1Yj3V45WpjzHYRWQlsNsa8CCwCHhcRg9W6+Vr46WnA2+GedRvWYZcJeyWNiy66iPb2dioqKigrK+MLX/gCn/vc55g7dy719fXMmjXrrM+///77ue+++5g9ezazZ89m/vz5AFx88cXU1dUxa9YsqqqqWLBgwcBzli9fzpIlSygvL2fDhg0Dy+fNm8eyZcu49NJLAWtnbF1d3bi3aVo7+/A0+vF4/Xh8VrC3dFoj9ax0FzWV+dy3YBrV5VaoTyvKJjVZJrXqOAKvPmKFTl4l3PkLmP254UM2FISetogvg1bo8p/+5RC5/Oiek8v7Os/++q50mDARLvws1H0RKuudE/yntGkug+tXnb1NE62siXDdv8AlX7a+QF5/FDb/HK59BObcFvefn05TnMRG+rn2BUN8eLAdj68Ft8+Px+dnb/MJwPp3PmNSDnVVhdROKaC2qiBhp3sdtcg2TV8nfOqv4aoHIT17+OfGQqD31C+J7pbTvzRafbDrZau+4guh7l6rrZE7efjXj0ddLdbnvfkZ60vsL74TfZvmXOx909phe/gDqLwEPvs4VF0yNu8VJZ2mWI2YMYam1m483pMtmA8OtNITsFowxTnp1FYVctu8SuqqCphbma/HpwN434P//tuTbZrrvw/FM8a3htR0yCmx/pxNTztsfwHcv7RGqa89ao3ya79g3boS4O9zoE3zCHQdH12bZiTOWwh/9SZ4noc3vgPPXAtzbrdG+AXDHDloAw16BUBHT4CtjdYo3eP14/b5aW63Tj5KT01hTnke914+ldoqa7ReWZhYl1Ibc6e1aZ61zrSM588oI9fauTjvL+HoR1bgb/mVdbRJdgnU3GWN9CfF6W/TB7fAfz8Ijf8TbtO8EJs2TbRSXDDvi3DRLfCnf4N3noSdL8EVX7N6+BnxcwW4hGndzJo1S4MlRowxdPcF2b5jJ2s/Nnh8fnYfbh+YJmB6cfZAoNdNKWBWaR7pqUnSVx8pu9s0sRYMwJ7XwPNLq7UTCkDFfCvw59xm7dC123i3aaLV2givr4St/2F9US7+R2sfSMr4nONxttZNQgT9J598Qm5uLkVFRRr256AvGKKzN0hnb4Cu3iAnegL0nmjl/Y8P8sNNbQOhXjulgNrKAgqz9YzSqHjfs0aUhz+wr00zlk4ctULL/Us4ssM6Wqf6Jqu1M+3K8Q9Wu9o0I9XYAOv/AXzvwaSLrOPvz796zN824YO+r6+PxsbGUR1bnkyMMXT1WVcc6g2EBs4qFSDNJaSlpoArjUml5Zw/OV+/PEfqlDZNBSx5PP7bNKNhDDS5rcD/YK11+GfBFKi9F2rvGZ+e9GltmhgdTTNWjIEdv7f2ffj3w4zPWr95lMwcs7dM+KBXwzPG4Pb5+c3mRv5rSxPtPQEm52VQP20ideER+5yKfJ0qYDROa9N8Ha76u8Rt05yLvi7Y+V9Wa2fvRkCsHZN1X7ROMEqL8ZxK8dqmiVZfN/zP/4O3VkHvCbjkf8HCFZBdFPO30qB3sMNt3fzu/QOsbfDxcfMJMtNSuH5OGbfPr+Ty84rGf7qAUCh8DHjEsd7p2VBWO269yjHh9DbNuWjZb+28dT8HrV6rfz/3DqufX1Y7ut9wEqVNE60TR2HDP0PDzyE9Fxb+nTXfTmrszv7WoHeYnkCQ13YcYW2Djzd3NxMyUD+1kNvnV3JDTdnoDnM0xhp5DHVyzhlP3PFDV3hZTxswxL+pCYVw/mLrlPLzFyfO8dqD2zSf/WerT+3UNs25CIVg39tWa2fnixDotnrTdfdCzZ3WLJEjkWhtmpE4shNe+T+w51UonA6fWRndSXRR0KB3AGMM2w60sbbBx++3NOHv7KM0L5Pb5ldw27xKzivJOblxoOcMoTzEGZVDhXlomJOX03NOPa1+uHla2g/BntetozlOHLFeo7QGLrjW+lN1afwds61tmnPT5Ydtv7VmijzQAClpMHOJ1do5/5qzT+aW6G2akdjzGqz/R2jeCVMXWDtsy+tG9ZIa9IkmFBwI4JbjR3l3+8e8v2sf7a3HKEzppK5EmFNkKMvoJaVniNAODLPTOjXz1JCOZmKtzHxrVJ6Rd+4zL4ZC1olEe161gt/3Z+tLJSPP6vNecK0VBgVVw7/WWNI2TWwc3mEF/pZfQ+dRyCm1dt7W3gvFF5zcLhSyWkCvPhxu03zZmjgsUds00QoGwP2s9eXWeRQuvgeueRjyys/p5ZIj6DuPww/rwZVhnRl4ym2GNb/HKbdDbTfS7c+wXYoLejuGn5/kTK2R3vaz/6ziii6YT5ncKmJdvEzE1N0Kn7xljW4+es2a2AugZFZ4tH8NTPnU+NWrbZqxEeiFj9ZbvfyPXgETtKYLrrvXmn7h1f9jfek7rU0Tre42ePtf4b0fW+2cr/35nP7NJUfQd7fBa/8EwR7rH9aQtz0Q7D3Dbc+wbxFTGXmnjaT9oQl86E/B02w41JuJycxnznlVXF59PlXlZRHT1GY7L3yMgeZdVujveQ32/8n6u0nLso7Z7g/+sbiAczBgTWX7xndPtmmufBAycoZ/rhqZ9kPWCN/9Szj2kbUsq9jZbZpoteyDtoMw9YpzenpyBP1oGQPBvii+GKL4AgkFrNOfz9QWycgbOAKl5UQvL25p4jcNPrYdaCPNJVw7ezJ31Fdy1YyS5JnpcbDeE7DvT+HgfxWO77WWF04/2duffuXoe+antGkWWSNKbdOMPWOsGSYPbrGO1HF6m2YcaNDHmUAwxNsfHeU3DT5e23GE3mCIi8rzuGN+JUtrK5ioZ6ae7tjH8PEbVvB/8pY18nalw9RPnQz+klnR/6bT0WxdVs7znLZplCNo0MeJPUfa+U1DIy+8f4Aj7T1MzE7n5toKbp9fSXV5nt3lJY5AD+x/Jzzaf906cgGswL7gGusQzvMWDj0vi7ZplENp0NuotauPl7Y0sbahEY/PjytFuHrmJO6or+TqmZN0srBYaG08efjm3o3WsfzisnbuXXCNNdovrbFaBev+1rq4/HmL4LrvQ8mFNhevVGxo0I+zYMjwpz1HWdvQyPrth+gJhJg5OZc76iu5qbbCWddCjTfBPivQ+3fqHtxiLc8qgs5j4TbNY1B9s7ZplKPohUfG0eG2bu746bt4j3eSPyGNuy6p4o75VcypyNPJw8aDK83q20/9lHVMcscRq7f/8QZr8q0F39A2jUo6GvQx9sqOw3iPd7Lqjou5saZMJxGzW84k68LOF99tdyVK2UaDPsY8Xj/FOencNq9CR/BKqbigewJjzO1robaqUENeKRU3NOhjqLWzj73NJ6iboid/KKXihwZ9DHka/QDUVWnQK6XihwZ9DHm8fkRgbmUcXEBZKaXCNOhjyO1r4cJJuaO78IdSSsWYBn2MGGPw+PzUattGKRVnNOhjZN+xTvydfbojVikVdzToY8TjawGgVoNeKRVnogp6EVkiIrtEZI+IrBhi/VQReV1EtorIRhGpjFj3f0Vku4jsFJEnxaEHmLu9frLTXcyYlGt3KUopdYphg15EXMCPgOuAauAeEaketNkq4FljTA2wEng8/NxPAQuAGmAOcAmwMGbVxxGPz09NZQGuFEd+jymlElg0I/pLgT3GmL3GmF7g18BNg7apBt4I398Qsd4AmUA6kAGkAYdHW3S86e4LsqOpTfvzSqm4FE3QVwC+iMeN4WWRtgC3hu/fAuSKSJEx5l2s4D8Y/rPeGLNz8BuIyHIR2Swim5ubm0f6M9hue1MrgZDRI26UUnEpVjtjHwQWiogbqzVzAAiKyAXAbKAS68thsYhcOfjJxpinjDH1xpj6kpKSGJU0ftxe64xY3RGrlIpH0cxeeQCoinhcGV42wBjTRHhELyI5wG3GGL+IfAV4zxjTEV73MnAF8HYMao8bbp+fioIJTMrNtLsUpZQ6TTQj+k3ADBGZLiLpwN3Ai5EbiEixiPS/1kPA6vB9L9ZIP1VE0rBG+6e1bhKdx+vX/rxSKm4NG/TGmADwdWA9VkivMcZsF5GVIrI0vNkiYJeI7AYmA4+Fl68FPgY+wOrjbzHGvBTbH8FeR9q6OeDv0v68UipuRXXhEWPMOmDdoGUPR9xfixXqg58XBP5qlDXGNbcvPGPllEKbK1FKqaHpmbGj5PH5SXMJF5Xn2V2KUkoNSYN+lNzeFqrL8vTasEqpuKVBPwrBkGFrY6v255VScU2DfhR2H26nszeo/XmlVFzToB8FT3hHrI7olVLxTIN+FNzeFgqz0phalGV3KUopdUYa9KPQf0Uph868rJRyCA36c9Te3cdHRzq0P6+Uinsa9Odoa2Mrxmh/XikV/zToz5Hba1068GINeqVUnNOgP0cen5/zS7LJn5BmdylKKXVWGvTnwBiD2+vX/rxSKiFo0J+DxpYujp3o1f68UiohaNCfg/fD/Xmdg14plQg06M+Bx+dnQpqLmZNz7S5FKaWGpUF/DtxeP3Mr80l16cenlIp/mlQj1BMIsqOpjTrtzyulEoQG/QjtaGqjNxjS/rxSKmFo0I/QyRkr9dBKpVRi0KAfIbfXT1l+JqX5mXaXopRSUdGgH6H+GSuVUipRaNCPwLGOHrzHO7U/r5RKKBr0I6D9eaVUItKgHwG3148rRZhbkW93KUopFTUN+hHw+PzMKs1lQrrL7lKUUipqGvRRCoUMW3x+7c8rpRKOBn2UPm7uoL0noP15pVTC0aCPkttr7YjVEb1SKtFEFfQiskREdonIHhFZMcT6qSLyuohsFZGNIlIZXn61iHgi/nSLyM2x/iHGg9vnJy8zlelF2XaXopRSIzJs0IuIC/gRcB1QDdwjItWDNlsFPGuMqQFWAo8DGGM2GGNqjTG1wGKgE3glhvWPG7e3hdophaSkiN2lKKXUiEQzor8U2GOM2WuM6QV+Ddw0aJtq4I3w/Q1DrAe4HXjZGNN5rsXa5URPgN2H2/WMWKVUQoom6CsAX8TjxvCySFuAW8P3bwFyRaRo0DZ3A78a6g1EZLmIbBaRzc3NzVGUNL62NrYSMtqfV0olpljtjH0QWCgibmAhcAAI9q8UkTJgLrB+qCcbY54yxtQbY+pLSkpiVFLsDJwRW6lBr5RKPKlRbHMAqIp4XBleNsAY00R4RC8iOcBtxhh/xCZ3Ai8YY/pGV6493N4WphdnU5idbncpSik1YtGM6DcBM0RkuoikY7VgXozcQESKRaT/tR4CVg96jXs4Q9sm3hljcOuMlUqpBDZs0BtjAsDXsdouO4E1xpjtIrJSRJaGN1sE7BKR3cBk4LH+54vINKzfCN6MaeXjpKm1m+b2Hu3PK6USVjStG4wx64B1g5Y9HHF/LbD2DM/dx+k7bxOGx9s/Y6UGvVIqMemZscNwe1vISE1hVmme3aUopdQ50aAfhsfnZ05FPump+lEppRKTptdZ9AVDfHCglTpt2yilEpgG/Vl8eLCdnkCIWt0Rq5RKYBr0Z+H2tQBQN0WnJlZKJS4N+rPweP2U5GZQnp9pdylKKXXONOjPwu3zU1dVgIjOWKmUSlwa9GfQcqKXT46e0P68UirhadCfgacxfEUpvXSgUirBadCfgcfrJ0WgpjLf7lKUUmpUNOjPwO3zc+HkXLIzopolQiml4pYG/RBCIcMWn18nMlNKOYIG/RA+OXaC1q4+nchMKeUIGvRD6J+xUk+UUko5gQb9ENy+FnIyUjm/JMfuUpRSatQ06Ifg8fm5uCofV4qeKKWUSnwa9IN09QbZebBd+/NKKcfQoB9kW1MrwZDRE6WUUo6hQT+I22vNWKlTHyilnEKDfhCPz0/VxAkU52TYXYpSSsWEBv0gbq+fWm3bKKUcRIM+wqHWbg62duulA5VSjqJBH8Hj0/68Usp5NOgjuH1+0l0pXFSeZ3cpSikVMxr0EdxeP7PL88hIddldilJKxYwGfVggGOKDxlbtzyulHEeDPmzX4Xa6+oI6NbFSynGiCnoRWSIiu0Rkj4isGGL9VBF5XUS2ishGEamMWDdFRF4RkZ0iskNEpsWu/Njx+PTSgUopZxo26EXEBfwIuA6oBu4RkepBm60CnjXG1AArgccj1j0LfN8YMxu4FDgSi8Jjze31MzE7naqJE+wuRSmlYiqaEf2lwB5jzF5jTC/wa+CmQdtUA2+E72/oXx/+Qkg1xrwKYIzpMMZ0xqTyGPP4/NRVFSCiM1YqpZwlmqCvAHwRjxvDyyJtAW4N378FyBWRIuBCwC8ivxMRt4h8P6xUqPoAAAqvSURBVPwbQlxp7epjz5EOnbFSKeVIsdoZ+yCwUETcwELgABAEUoErw+svAc4Dlg1+sogsF5HNIrK5ubk5RiVFb2ujXlFKKeVc0QT9AaAq4nFleNkAY0yTMeZWY0wd8O3wMj/W6N8TbvsEgP8E5g1+A2PMU8aYemNMfUlJyTn+KOfO7fUjAjVV+eP+3kopNdaiCfpNwAwRmS4i6cDdwIuRG4hIsYj0v9ZDwOqI5xaISH96LwZ2jL7s2PL4/FxQkkNeZprdpSilVMwNG/ThkfjXgfXATmCNMWa7iKwUkaXhzRYBu0RkNzAZeCz83CBW2+Z1EfkAEODpmP8Uo2CMwe1t0f68UsqxUqPZyBizDlg3aNnDEffXAmvP8NxXgZpR1DimvMc7aens0/68Usqxkv7M2P4TpXREr5RyqqQPerfXT1a6iwsn59hdilJKjQkNep+fuRX5pLqS/qNQSjlUUqdbd1+QHU2t2p9XSjlaUgf9joNt9AWN9ueVUo6W1EHv9vafEatBr5RyrqQOeo/PT3l+JpPzMu0uRSmlxkxSB73b26L9eaWU4yVt0De399DY0qX9eaWU4yVt0A9cUUr780oph0vioG8hNUWYU6EzViqlnC1pg97t9TO7LI/MtLi7DopSSsVUUgZ9MGTY2tiq/XmlVFJIyqDfc6SDjp6A9ueVUkkhKYPe42sBdMZKpVRySMqgd3v95E9IY3pxtt2lKKXUmEvKoPf4/NRWFSAidpeilFJjLumCvqMnwK7D7dqfV0oljaQL+q2NfozR/rxSKnkkXdD3z1ipQa+UShZJF/Qen5/zirMpyEq3uxSllBoXSRX0xhjcXj+12p9XSiWRpAr6A/4ujnb0UKdtG6VUEkmqoD95RSmdg14plTySKug9Pj8ZqSnMLM21uxSllBo3SRX0bm8LNZX5pLmS6sdWSiW5pEm83kCIbU1telilUirpJE3Q7zzYRm8gpP15pVTSiSroRWSJiOwSkT0ismKI9VNF5HUR2SoiG0WkMmJdUEQ84T8vxrL4kei/dKCO6JVSySZ1uA1ExAX8CPgM0AhsEpEXjTE7IjZbBTxrjPl3EVkMPA58MbyuyxhTG+O6R8ztbWFyXgZl+Zl2l6KUUuMqmhH9pcAeY8xeY0wv8GvgpkHbVANvhO9vGGK97XTGSqVUsoom6CsAX8TjxvCySFuAW8P3bwFyRaQo/DhTRDaLyHsicvOoqj1Hx0/0su9Yp/bnlVJJKVY7Yx8EFoqIG1gIHACC4XVTjTH1wOeBJ0Tk/MFPFpHl4S+Dzc3NzTEq6aQt2p9XSiWxaIL+AFAV8bgyvGyAMabJGHOrMaYO+HZ4mT98eyB8uxfYCNQNfgNjzFPGmHpjTH1JScm5/Bxn5fa2kCJQU5kf89dWSql4F03QbwJmiMh0EUkH7gZOOXpGRIpFpP+1HgJWh5cXikhG/zbAAiByJ+64cPv8zCzNIyt92H3PSinlOMMGvTEmAHwdWA/sBNYYY7aLyEoRWRrebBGwS0R2A5OBx8LLZwObRWQL1k7a7w06WmfMhUIGj8+vV5RSSiWtqIa4xph1wLpByx6OuL8WWDvE894B5o6yxlHZe/QE7d0B7c8rpZKW48+MdXtbAJinI3qlVJJyfNB7fH5yM1M5rzjH7lKUUsoWjg96t9c6USolRU+UUkolJ0cHfWdvgF2H27U/r5RKao4O+g8aWwmGjB5xo5RKao4O+v4ZKy+u1KBXSiUvRwe92+tnalEWRTkZdpeilFK2cXTQ989YqZRSycyxQX+wtYtDbd3UadArpZKcY4Pe4w3PWKlTEyulkpxjg97t85OemkJ1WZ7dpSillK0cG/Qer5+LyvNIT3Xsj6iUUlFxZAr2BUNsPeCnrkrbNkop5cig33Wone6+ELV6opRSSjkz6N3hE6X0iBullHJo0Hu8fopz0qksnGB3KUopZTtHBr3b10JtVSEiOmOlUko5LuhbO/vY23xCJzJTSqkwxwW9p1H780opFcl5Qe/1IwJzK/PtLkUppeKC44Le7Wvhwkm55Gam2V2KUkrFBUcFvTFGZ6xUSqlBHBX0+4514u/s0x2xSikVwVFB7/G1AOgZsUopFcFRQe/2+slOdzFjUq7dpSilVNxwVNB7fH5qKgtwpeiJUkop1c8xQd/dF2RHU5v255VSahDHBH17d4Dr55ax4IJiu0tRSqm4ElXQi8gSEdklIntEZMUQ66eKyOsislVENopI5aD1eSLSKCI/jFXhg5XkZvDkPXUa9EopNciwQS8iLuBHwHVANXCPiFQP2mwV8KwxpgZYCTw+aP13gLdGX65SSqmRimZEfymwxxiz1xjTC/wauGnQNtXAG+H7GyLXi8h8YDLwyujLVUopNVLRBH0F4It43BheFmkLcGv4/i1ArogUiUgK8K/Ag2d7AxFZLiKbRWRzc3NzdJUrpZSKSqx2xj4ILBQRN7AQOAAEga8C64wxjWd7sjHmKWNMvTGmvqSkJEYlKaWUAkiNYpsDQFXE48rwsgHGmCbCI3oRyQFuM8b4ReQK4EoR+SqQA6SLSIcx5rQdukoppcZGNEG/CZghItOxAv5u4PORG4hIMXDcGBMCHgJWAxhjvhCxzTKgXkNeKaXG17CtG2NMAPg6sB7YCawxxmwXkZUisjS82SJgl4jsxtrx+tgY1auUUmqExBhjdw2nqK+vN5s3b7a7DKWUSigi0mCMqR9yXbwFvYg0A/tH8RLFwNEYlZPo9LM4lX4ep9LP4yQnfBZTjTFDHs0Sd0E/WiKy+UzfaslGP4tT6edxKv08TnL6Z+GYuW6UUkoNTYNeKaUczolB/5TdBcQR/SxOpZ/HqfTzOMnRn4XjevRKKaVO5cQRvVJKqQga9Eop5XCOCfrhLo6STESkSkQ2iMgOEdkuIt+wuya7iYhLRNwi8l9212I3ESkQkbUi8qGI7AzPSZW0RORb4f8n20TkVyKSaXdNseaIoI/y4ijJJAD8rTGmGrgc+FqSfx4A38CawkPBvwF/MMbMAi4miT8XEakAHsCah2sO4MKaz8tRHBH0RHdxlKRhjDlojHk/fL8d6z/y4GsIJI3wpS1vAH5mdy12E5F84CrgGQBjTK8xxm9vVbZLBSaISCqQBTTZXE/MOSXoo7k4SlISkWlAHfBneyux1RPA/wZCdhcSB6YDzcDPw62sn4lItt1F2cUYcwDrUqhe4CDQaoxx3NXwnBL0agjhawP8FvimMabN7nrsICI3AkeMMQ121xInUoF5wE+MMXXACSBp92mJSCHWb//TgXIgW0Tutbeq2HNK0A97cZRkIyJpWCH/nDHmd3bXY6MFwFIR2YfV0lssIr+0tyRbNQKNxpj+3/DWYgV/sroW+MQY02yM6QN+B3zK5ppizilBP3BxFBFJx9qZ8qLNNdlGRASrB7vTGPMDu+uxkzHmIWNMpTFmGta/izeMMY4bsUXLGHMI8InIzPCia4AdNpZkNy9wuYhkhf/fXIMDd05Hc4WpuGeMCYhI/8VRXMBqY8x2m8uy0wLgi8AHIuIJL/sHY8w6G2tS8eOvgefCg6K9wH0212MbY8yfRWQt8D7W0WpuHDgdgk6BoJRSDueU1o1SSqkz0KBXSimH06BXSimH06BXSimH06BXSimH06BXSimH06BXSimH+/+erZievdvqpQAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"lFshsHQJ6E41"},"source":["Training is rather unstable; accuracy falling down after around 80 epochs"]},{"cell_type":"code","metadata":{"id":"bzkXzHRv6E42","executionInfo":{"status":"ok","timestamp":1604776109469,"user_tz":420,"elapsed":16098,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"ecd763bc-e488-4aa8-c2b3-fa772bccff81","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":23,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 15s 48ms/step - loss: 0.0451 - accuracy: 0.9872\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"iWbs-sWK6E44","executionInfo":{"status":"ok","timestamp":1604776109470,"user_tz":420,"elapsed":16091,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"784dfa21-ef08-441d-f904-b369d9afc771","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":24,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9872000217437744\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"collapsed":true,"id":"N-ZZBZJQ6E45"},"source":["## 2. Deep CNN - 2\n","- Measures used to ameliorate the performance of MLPs are employed\n","    - Weight Initialization\n","    - Batch Normalization\n","    - Dropout"]},{"cell_type":"code","metadata":{"id":"o4gBcJAK6E46","executionInfo":{"status":"ok","timestamp":1604776109470,"user_tz":420,"elapsed":16087,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["from tensorflow.keras.layers import BatchNormalization, Dropout"],"execution_count":25,"outputs":[]},{"cell_type":"code","metadata":{"id":"3DhUJW0S6E48","executionInfo":{"status":"ok","timestamp":1604776109471,"user_tz":420,"elapsed":16085,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def deep_cnn_advanced():\n","    model = Sequential()\n","    \n","    model.add(Conv2D(input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]), filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    \n","    # prior layer should be flattend to be connected to dense layers\n","    model.add(Flatten())\n","    # dense layer with 50 neurons\n","    model.add(Dense(50, activation = 'relu', kernel_initializer='he_normal'))\n","    model.add(Dropout(0.5))\n","    # final layer with 10 neurons to classify the instances\n","    model.add(Dense(10, activation = 'softmax', kernel_initializer='he_normal'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model    "],"execution_count":26,"outputs":[]},{"cell_type":"code","metadata":{"id":"jehCWNZT6E4_","executionInfo":{"status":"ok","timestamp":1604776109669,"user_tz":420,"elapsed":16281,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["model = deep_cnn_advanced()"],"execution_count":27,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"ibjgkkls6E5D","executionInfo":{"status":"ok","timestamp":1604776109670,"user_tz":420,"elapsed":16274,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"757afe35-8654-480e-ef2e-292411b8e7d7","colab":{"base_uri":"https://localhost:8080/"}},"source":["model.summary()"],"execution_count":28,"outputs":[{"output_type":"stream","text":["Model: \"sequential_2\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","conv2d_7 (Conv2D)            (None, 28, 28, 50)        500       \n","_________________________________________________________________\n","batch_normalization (BatchNo (None, 28, 28, 50)        200       \n","_________________________________________________________________\n","activation_7 (Activation)    (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","conv2d_8 (Conv2D)            (None, 28, 28, 50)        22550     \n","_________________________________________________________________\n","batch_normalization_1 (Batch (None, 28, 28, 50)        200       \n","_________________________________________________________________\n","activation_8 (Activation)    (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","max_pooling2d_4 (MaxPooling2 (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","conv2d_9 (Conv2D)            (None, 14, 14, 50)        22550     \n","_________________________________________________________________\n","batch_normalization_2 (Batch (None, 14, 14, 50)        200       \n","_________________________________________________________________\n","activation_9 (Activation)    (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","conv2d_10 (Conv2D)           (None, 14, 14, 50)        22550     \n","_________________________________________________________________\n","batch_normalization_3 (Batch (None, 14, 14, 50)        200       \n","_________________________________________________________________\n","activation_10 (Activation)   (None, 14, 14, 50)        0         \n","_________________________________________________________________\n","max_pooling2d_5 (MaxPooling2 (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","conv2d_11 (Conv2D)           (None, 7, 7, 50)          22550     \n","_________________________________________________________________\n","batch_normalization_4 (Batch (None, 7, 7, 50)          200       \n","_________________________________________________________________\n","activation_11 (Activation)   (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","conv2d_12 (Conv2D)           (None, 7, 7, 50)          22550     \n","_________________________________________________________________\n","batch_normalization_5 (Batch (None, 7, 7, 50)          200       \n","_________________________________________________________________\n","activation_12 (Activation)   (None, 7, 7, 50)          0         \n","_________________________________________________________________\n","max_pooling2d_6 (MaxPooling2 (None, 3, 3, 50)          0         \n","_________________________________________________________________\n","flatten_2 (Flatten)          (None, 450)               0         \n","_________________________________________________________________\n","dense_4 (Dense)              (None, 50)                22550     \n","_________________________________________________________________\n","dropout (Dropout)            (None, 50)                0         \n","_________________________________________________________________\n","dense_5 (Dense)              (None, 10)                510       \n","=================================================================\n","Total params: 137,510\n","Trainable params: 136,910\n","Non-trainable params: 600\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"gYaFxIts6E5F","executionInfo":{"status":"ok","timestamp":1604779627120,"user_tz":420,"elapsed":3533716,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"1f6b59ef-2f89-4f11-e289-8da9a2763605","colab":{"base_uri":"https://localhost:8080/"}},"source":["%%time\n","history = model.fit(X_train, y_train, batch_size = 50, validation_split = 0.2, epochs = 10, verbose = 1)"],"execution_count":29,"outputs":[{"output_type":"stream","text":["Epoch 1/10\n","960/960 [==============================] - 348s 363ms/step - loss: 0.3679 - accuracy: 0.8820 - val_loss: 0.0768 - val_accuracy: 0.9783\n","Epoch 2/10\n","960/960 [==============================] - 349s 364ms/step - loss: 0.1215 - accuracy: 0.9636 - val_loss: 0.0327 - val_accuracy: 0.9897\n","Epoch 3/10\n","960/960 [==============================] - 350s 365ms/step - loss: 0.0900 - accuracy: 0.9732 - val_loss: 0.0579 - val_accuracy: 0.9843\n","Epoch 4/10\n","960/960 [==============================] - 350s 365ms/step - loss: 0.0739 - accuracy: 0.9776 - val_loss: 0.0665 - val_accuracy: 0.9821\n","Epoch 5/10\n","960/960 [==============================] - 350s 365ms/step - loss: 0.0652 - accuracy: 0.9808 - val_loss: 0.0438 - val_accuracy: 0.9889\n","Epoch 6/10\n","960/960 [==============================] - 355s 370ms/step - loss: 0.0567 - accuracy: 0.9825 - val_loss: 0.0603 - val_accuracy: 0.9846\n","Epoch 7/10\n","960/960 [==============================] - 349s 364ms/step - loss: 0.0484 - accuracy: 0.9856 - val_loss: 0.0367 - val_accuracy: 0.9912\n","Epoch 8/10\n","960/960 [==============================] - 355s 370ms/step - loss: 0.0456 - accuracy: 0.9862 - val_loss: 0.0319 - val_accuracy: 0.9912\n","Epoch 9/10\n","960/960 [==============================] - 352s 367ms/step - loss: 0.0400 - accuracy: 0.9880 - val_loss: 0.0332 - val_accuracy: 0.9925\n","Epoch 10/10\n","960/960 [==============================] - 353s 368ms/step - loss: 0.0377 - accuracy: 0.9881 - val_loss: 0.0349 - val_accuracy: 0.9904\n","CPU times: user 1h 46min 37s, sys: 2min 51s, total: 1h 49min 28s\n","Wall time: 58min 37s\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"GB-i4R2g6E5H","outputId":"f7fc65b4-5962-4544-b7bf-4d8ee13add35"},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.legend(['training', 'validation'], loc = 'upper left')\n","plt.show()"],"execution_count":null,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8nOWZ7//PNU1dVrVcZCwDxhU3hIHQYwI2ECCEmsMm\nkBCn0JKTPRtCzoY9u8lr+WWzWSCQeA0hySaAQwgkQEyvIQHjgnFv2MaWiyQX9TLt+v1xj+WxrJHG\ntozMM9f79fLLmqfM3Pdo9H3u53rKiKpijDEmc/gGugHGGGM+Xhb8xhiTYSz4jTEmw1jwG2NMhrHg\nN8aYDGPBb4wxGcaC3xhjMowFvzHGZBgLfmOMyTCBgW5AT8rKyrSqqmqgm2GMMZ8Yixcv3qWq5eks\ne0wGf1VVFYsWLRroZhhjzCeGiHyU7rJW6jHGmAxjwW+MMRnGgt8YYzJMnzV+EXkEuBSoU9WJPcwX\n4D7gYqANuFFVlyTmzUzM8wMPq+o9h9vQSCRCTU0NHR0dh/sUJkl2djaVlZUEg8GBboox5mOWzsHd\nXwMPAP+TYv4sYHTi32nAL4DTRMQPPAh8BqgBForIM6q66nAaWlNTQ0FBAVVVVbhtjTlcqsru3bup\nqalh1KhRA90cY8zHrM9Sj6q+BezpZZHLgf9R512gSESGAtOBDaq6UVXDwLzEsoelo6OD0tJSC/1+\nICKUlpba3pMxGao/avzDga1Jj2sS01JNP2wW+v3H3ktjMtcxcx6/iMwGZgMcd9xxA9waY4zpX9FY\nnDU7m3l/awN5IT8XTRhCXtbARHB/vOo2YETS48rEtGCK6T1S1bnAXIDq6upj7ouAGxoaeOyxx/jm\nN795SOtdfPHFPPbYYxQVFaVc5gc/+AHnnHMOF1xwwZE205gurZ1RXlldS0VhNuOHFVKYvf9Avqr2\nudfX2BahqSOCzyf4BAqyg+R3CypVRRV8vvT2IOuaOvjbh7sI+f3kZwcoyA4wuCCL8oIssgJ+OqMx\ntjd0UNvUQXFuiBElOeSGAqyrbea5D7bz0qpaSvJCzBhXwYyxgynKDVKzt51tDe10RGL4fULA56Ms\nP8S4oYVdwbqxvoXX1tSxvraFYEAI+f3kZfkZXJhNRUEWxXkhwtE47eEYbZEYDW1h9rZGaE703594\nD/a0RtjV0klDW5iJwwfxmfEVTK8qIeB3xZOOSIz1tS0s39bIyu2N1Dd30twRpbkzwsb6VtrCsa73\nIi+0gksnDeOC8RUMHZRNRWE2pXmhtN/LIyHpfNm6iFQBz6U4q+cS4FbcWT2nAfer6nQRCQDrgBm4\nwF8IfEFVV/b1etXV1dr9yt3Vq1czbty4Ptt6tGzevJlLL72UFStWHDA9Go0SCBwzO06HZKDf00wX\nicVRhVDg4IprLK5s2dPGutpmmtojVBbnMqIkh4DPx8LNe3hv0x42725lZGkuJ5bnc8LgfE4cnM+Q\nwmzCsTiPL9jCz17bwO7WcNdzVhbnoAqN7RFaOqOMKsujemQxp4wsJivoY29rhL1tYTbUtbBieyNb\n97Qf1K6CrABDBmWTFfSxpyXM7tYwkVicQTlBinNDlOSFKE8EeXl+FmWJ/zuiMZ5eso031tUTi/ec\nOQXZAZo7oimn+wSqq0rY2xpmfV1Ln++vCJxQnk8srmza1QpAWX4WsXjchXwkRoqmdMkN+VGFaDxO\nLK6U5IUoy88iLyvA8m2NhKNxCrID5AT9NLZH6IzGu9YtzA4wrCiHguwA+VkBjivJZdrIYqYdV0xt\nUwdPLNrKc8t2HLAxKM0LsfifP9Nn33ruryxW1ep0lk3ndM7HgfOAMhGpAe7GjeZR1TnAfFzob8Cd\nznlTYl5URG4FXsSdzvlIOqF/rLrzzjv58MMPmTJlCsFgkOzsbIqLi1mzZg3r1q3jiiuuYOvWrXR0\ndHDHHXcwe/ZsYP/tJ1paWpg1axZnnXUWf//73xk+fDh//vOfycnJ4cYbb+TSSy/lqquuoqqqii99\n6Us8++yzRCIR/vCHPzB27Fjq6+v5whe+wPbt2znjjDN4+eWXWbx4MWVlZQP8zph9Gtsj+H1CbtCP\nzyfE4sru1k7qmztd2HZEaeqIsq62mSUf7WX5tkY6o3HK8rMYVpRNdtBPc0eUls4IdU2dB4RId7kh\nP6PK8li6teGAsMwL+ckO+tndGub040v42YzRhKNxVm5vYs3OZoJ+YVBOkLxQgLW1zbyyupY/LK7p\nWl8ERpbkMqmyiOunH0d5fhaqEFOlqT3CjsYOtje0E4nFGVNRSGl+iKBfaGiL0NAeYU+L23C8s3E3\nDW2RA9o8uCCL2eccz6WThhLw+WjpjNDYHqG+uZPapk52t3RSkpfF8OIchhRms7ctzNa9bWzb287Y\nIQVcNHEIgwuyAdiyu4031tURjsYZXpTD8OIc8rMCRONKJBZnR0MHK7Y3smJbI7G4ctOZVZw/ZjAj\nSnK72hOLK7tbOtnZ1EFDW4TsoJ/soI/ckJ9BOSGKcoME/akPg7Z2Rvnr+l28ua4eVWVQTpDCnCCj\nyvKYOGwQI0pyUu5VjSjJpbqqhLs/O4F1tc3UNnVS29RBJJb6d96f0hrxf9z6GvH/v2dXsmp7U7++\n5vhhhdz92Qkp5yeP+N944w0uueQSVqxY0XU65J49eygpKaG9vZ1TTz2VN998k9LS0gOC/8QTT2TR\nokVMmTKFa665hssuu4wbbrjhoOD/zne+w2233cbPf/5zlixZwsMPP8ytt97K8OHD+d73vscLL7zA\nrFmzqK+vP6Lg/6SO+ONxZUN9C5XFrgzQn3Y2dvDqmlpeW13H2tpmJlcWcfrxJUwZUUxHNMbe1jAN\n7RFaO6O0hWM0tUdYV9vMqh1N1DZ1Ai4880IB2sLRHkeUIb+PicMLmXZcMQXZQXY0trO9sYPOSIyC\n7CAF2QHK8kOMrijgpIoCinKCbGtoZ8ueNjoiMaYdV8yEYYUE/D5UlfqWTjbUtfBhfSsf1rVQ39LJ\ntdUjOHt0WZ/lHFXlo91txFUpzg1RmBPE30+lhs5ojN0tYeqbO4nE4kwZUdRVEjH9r19H/KZn06dP\nP+Ac+Pvvv5+nn34agK1bt7J+/XpKS0sPWGfUqFFMmTIFgFNOOYXNmzf3+NxXXnll1zJPPfUUAG+/\n/XbX88+cOZPi4uJ+7c+AikWh4SMoPSHlIo3tEVZua+Tl1bW8sGInOxo7qCzO4T+umswZJ7j3eW9r\nmMcXbqFmbzshv4+sgI/CnCAVhdlUFGYR9PvY0xruGuVt3dPO1r1t1DV10hqO0tYZI5wYcY0oyWHi\nsEEs/mgvf1m+I2W7gn7hhPJ8zjyhjJOGFOATaOmI0tbeSm5OHuUFWZTlZzEoN0hhokY+ZJAb3R+K\nqrI8zuxhuogwuCCbwQXZfOqEQx8EiAhVZXmHvF46sgJ+hhXlMKwo56g8vzl8n8jg721k/nHJy9v/\nx/LGG2/wyiuv8M4775Cbm8t5553X4znyWVlZXT/7fT7aI5GDlklezu/3E40eXPNMi8YhHkuaIODz\nu+HooWqpg0AWZA9Kf532vbDiKfjgcbf+RT+CcZ+lrrmDDbUtfFjfQsuubZzV9BdO2vYUWW074Prf\nw5iZgBvVv7Gujsff28rumnWc3foK5/uXkqvTOfnEL/O1c47nN+98xPUPvcuNn6rCJ8K8hVtoC8co\nzQuRG2tgcmwlf42Ws0qrDmjaBNnEJP9mlhd8hqElgzhtVAn52QFyQ26kfe5J5Zw4OB8RQdXV2ldt\nbyIvK0BxrisB5GcFyM3yE/L7Dh5V71wOv7oCpt8MZ/zgwHkr/gjbIjDusxA6gsBt2wPrXoSqs6Ao\n6RyKljrYvQEqp4O/lz/vljrIKQZ/tyu3d3wA29+Hpu3QtA1C+TDkZKiYCIPHuc9Bf4jHobXe/csr\ng7zB4OthbyAeh7V/AfG7dgyqhFgEdq2FnSvAF3ADhtITIbtw/3qqEO2EaDsEc1O3u6MJ9nwInS1w\n3On73494zP2ulv8BysfC8efCsGmwbTGsewE2/w3Kx8BJM2H0hZBX2vPzJ9vXpkib+1vyHdrGvz99\nIoN/IBQUFNDc3NzjvMbGRoqLi8nNzWXNmjW8++67vT9ZtBNaaqGt0/2chjPPPJMnnniC7373u7z0\n0kvs3bvXrdu2ByLtEAu7P4h4BOJRF/zd+QIQyIFQDuSU9P6C8RhseBUWPgzrX3J/ECd8GiZ8jnh2\nCbXrF9K0aQktOUMZfMk/M2JohVsv3EbslX/Ft/gRJNZJvGwsEX8uWb+/gdeC5/IvLVdwmm81l/gW\ncJZvOQGJ81bsZEZJlI7ff5eHxlYwsryAp5bUUF9fx0M5D3CafoAGhbbC45nS9Dj4amHyXK45eRJ/\n/f1PGbPo2+RKJzfmVlB8fBUFHdtdKPgVLaxgy/96mx0dfndwLhtGP3kXgYbNkPcmzPgPqDrV9XHh\nL2HZIlh1nAuSiolI9U2MLC1mZGkipGNR+Ohv0NEAkQ73Po+ZBTmJs7aaa+Gx66CzCd7+LxhzCVSe\n4uatfwWe/LL7+bn/DROugKFTXDhF2qG4CiZ8bn9IqcKHr0JLPUy+bv9Gu7MFfvs52LHUPa6YCMOm\nwLYlUJe4MH7oFLjsfhg6+cDf69b34K2fwPoXXR8/868w5mL3eXz5B7Ds92458UF+hQvGiDswij/L\nPd+I6VA0MtHuDigcCpOuTW+jsPYFeOn7sPcj91lN/mwOqoQJV8L02e45a1fBs3dAzXv7l8seBOG2\nA9fteo7g/vcoFgESdbasQXDKl+C0r0HBMPjobVj6eOK9rd2/fm4ZnHw1DJ0Ef7sf6lfDoBHw4evw\n9/v3LxfMc+/Blndh1Z8AcRvIYI77N/JTUP0VqKx2v9dl8+DdObBr3f42FY+CKx+CEafuf97mWjdo\nGH30z+6z4E9TaWkpZ555JhMnTiQnJ4eKioqueTMvvJA5P3+AcePGMmbMWE4//fTUTxSLuBGZqgvo\n+jUuwONRF+LxGDTWQJ7fhUzC3XffzfXXX89vf/tbzjh1KkMGl1HQXgPxECDgD7lwDuW5PyJfwI2S\n9g1GVSHa4UYbLfVuxNfSBOu3uUBPHm1tfJPos98msPdDwtllbD7pq3R2tHHc5pcZtO4FfMBQIK5l\njOZVauf8hf8ovpXCsiF8dtMPGRbbzu+j5/Hb2AWsqBlFgBjfDDzD7TzNW1lvAhAtPA7fxNvYceI1\naLSC9Uuf4NOr7kJXP8N/LKlmwrBCnjvxWUZsWwHnfx+ZfD15gyph8a/h+e/CL84gN9LBReFm2itP\nI15yIoPbd0LTFjeCPP/7MGg48qdvMHLNQ4z89Pdd3xb8NzRshnP+jxvN/c/lkFsKbbshfwiMvdSN\ndmsWuRHf3++H874HU29wy7/9X7B384G/0/whcPGP3chv3hegfQ/c+Bz88avw51vga2+65396Ngye\nADP/3T3Xyj/B0kcPfK6XfwDTv+pGwO/+3H0+ADa8DJc94H7HT3zRBcRlP4P2BjfyX/UsDJ8Gk65x\nI/nXfgRzz088Vxk0boO61bD1XbfRP/MOWPu8a2/lqVC3BmKdcPZ34JQboWCoe614HPZuSuwJLHEb\njvcecssme/PHcN6dMOk6t6Fo2u4+6xUT93+23p0DL34PBo+HM25xQZ9XBq273PJ1q+Bv98LffwbH\nnwcb33Cj+CvmQMnxULscaldCVqEb/Q852W14d2+AXevdxnYfX8CFcCDHbTjeecC9n3mDoXk7hArc\nBnvwOCgb7dZZ/gdY9Es3iCo9Ea76FYy/wm3gtrwD25e6DWrVWRDMdu/Nzg/cAKlttwv5jgZY/azb\n062Y6PrVvsetd84/ur0PfxAWzIVHLoJzvwtjL3bvzfInXJu/s849/1H0iTy4e9Q17XC7oDlF7o8k\nlNdziUTjLqybd+4fgYTyE+uVHrzrGo+6D+i+D5YvAA1bIJx0apokdv80UabJKoTikXRGYvj9fgLR\nVt556Wm+cdf/x9IFb7m2BXIOrYQTi0LbLlavWsG45z8PRcfRPu4aXpdqSlb8mtObXmBzvIKfRK/h\nxfipRBLjg9ygcF7BVkaXhBg96TQ+NeFEYlsXIs/eQVnregDq/BW8PPoHaNXZRGNxwrE4uaEAF4yr\nYEj7ejeyPv48t9uc3OZ4DH5xJqox6m54g8HbXkb+8CX3h3H+XQe2f+cKeOFONyI94xYXeKk8+WVY\nMx9uWwxZ+XD/VKiYAF98xm1w3/mZC7WTr3Yj3+TSx45l8NL/hU1vutFkPALDpsKZ33K/v2CO24A+\n/0+wcxkUVkJTDVz7O1fKWfciPHYNnPW/YesCFxyz34Dyk9zzRzrc7z6YA4Fs2PSWC6gNr7j5FSfD\np251JZdX/82NtkuOh5VPwWfvd6PYVNr3uo3IksQttnJLXdCefDWccpN7L2IRtyH9230ujGf+e6/H\nWbpEO92eQDDbffY2vwWv/qsrEflD7vO9T+FwF56RVvdaYy+FK+emLnPt2QQL5riN7okXwIU/Sq+M\n0pe9H8F7c91Ge/wVMPYSCOUevFzbHrcBGnF676Wy3nQ2w7InYOljUDAETv+m2wtI/rx3NMJf/tGF\nPbj3cer/csum8zvowaEc3LXg725fzS+Q40Y1Gnd/lKWjD/wgRDrccrGw2/XLH+xGBu0NbmQdynd/\npPvqeLEI7NnoRgUlx++vR6q60UI85v4Yg4kPY6TNtaWlFvwh1u9Rrrn+C8QjHYRCIX4+5yFOPa2X\nPYtuorE4PpGui0PiqqxcuYr4lr8RWj6PMW1L8IkSxc+rJddSN+V2KgeXUpgToDA7SHlBFoNygj2f\nJRKLwII5aHMtct53IavgcN55WPUMPPEP8Jl/cyProhFw86sH16EPxd6P4IFTYeLnXfngr//pwnfY\n1PTWV3Ubq9XPuDLECZ8+eCMbi7rR5Js/hnP/jxtN7/PU7P3lk8/9tyvZ9KV+rQuGylP3v9ba5+GP\nN7sNxXnfc6PrdLTudgEXPMoHWFVhzV/go7+7sCsc5j4Xq59xG7JYGM641ZWWBrC2fcxZ/Zwb/E2+\nDnL7KL/2wYI/XZ0tLmBzS92HMRZ2f3S+AJSNAdSNnBprXJiVHO/+EONxd3ApHoWi49yoPDkM2va4\nX2YwB0pOcK/R8JHbiBSN3F8PTreNezbuf37xQdlJfYZhNBZPnBMepbUz2nW2yr7wj8Xi7Nyyka8+\ns4PSvBD/MD7ANYNWMnTieciQg67T+3iown+f40bP/iz42lsweOyRP+/LP3A1W38Ixl8Gn3/4yJ+z\nJ/H4wXt5bXtg7rlu9Hrpfx3Z89evcwcXk+v9nwQdjW6vuHzMQLfE0yz407V7g9st8wXdaLBtjwvp\nsjEH1tha6tzu9qBKyCt3od6224V68pkEyToa3W6rL+BKBIFsd/DucEZekQ4X/vGoq0cGc4jG4jS2\nR7quFswJ+skN+Qn4hcb2KC0dURQl4BPysgJJVyAq8bgSCPjYtmk9FA1nUmVRrxeqfKzWvwKPfh4u\n+nc449Buj5FSe4Mr8YRb4NZFUDyyf543XbHo4ZcNjEmTncefrmjn/tJKwxb3f9HIgw+s5JW7DUTj\ntkR9fLerL6cKfXBnH5Se6A6M5Za6WmeKXVxV7QrkuGrXRT/77qfi9wXxl55ELBalqQOaG1po64yh\nKFkBP3lZAdrDMZo63HGGkN9HWUGIopwg2UF/yot49oYCjBt5ZLuX/W70BfCdta5c0F9yiuD6eW7v\n7eMOfbDQN8eczP1EqroaZE6xO4Oho8GNqHuqs4m4kk79GmjZ6Wr6BUP7fo2sfHdkv5fd8nA0zqZd\nrXRGYymX6S4n6O+quWcH959HHo3FicaVrEAP55Z/kvRn6O9z3Gn9/5zGfEJlbvDHwoC6WrKI2wD0\nxh90pZrmnW6vIN1gTVqurTNKwO/ruilXOBpjY30rMVWGDcrB7xd8Ioi4szD3XUAUi7s9Ap8IBVkB\ngj3c1Asg4PcRsONmxpg+HCOF3QGw78KpQCj9dbIKXI09jXXy8/MB2L59O1dddRUNbWE21LewdmcT\nH+1upaEtzFnnnMcHS5cwqiyPsoIsinND7kZP2cGuW+D+cs6DhIhSlp9FSV6Iyy+7lIaGhsPpsTHG\nAF4N/kg77NrgRuep7LsApb8uQU9h2LBhPPK7x9m6p528rABlBVm0dkbZsqcNUIYXZfd6o7F7772X\ntra2rsfz58/v9d7+xhjTF28FfzzurpSrX+vO4Gje4Q7E9iQaxt2/Jr1zxO+8804efPDBrsf/8i//\nwg9/+ENmzJjBtGnTOPnkk/nzn/980Hqr1m7glCmTyQ76qMgVvv21m/j8jNO565tfgmiY7KAL/W98\n4xtUV1czYcIE7r77bsDd+G379u2cf/75nH/++YC7zfOuXbsA+OlPf8rEiROZOHEi9957L+DuIjpu\n3Di++tWvMmHCBC688ELa2w++r7oxJnN9Mmv8z9/pLlk/gLpTMTXuwjwQgkinuwI2mLP/ith9ou1u\n2WDiCsIhJ8Ose1K+5LXXXsu3vvUtbrnlFgCeeOIJXnzxRW6//XYKCwvZtWsXp59+OudeMJP2qPuC\njXU7m9mytw3B3V3xZ/fdS25uLqtXr2bZsmVMm7b/itMf/ehHlJSUEIvFmDFjBsuWLeP222/npz/9\nKa+//vpBt19evHgxv/rVr1iwYAGqymmnnca5555LcXEx69ev5/HHH+ehhx7immuu4Y9//CM33HDD\n4b7bxhiP8dCIX5Luz5EN+NxpmSKu9NP9pmUadxdDpWnq1KnU1dWxfft2PvjgA4qLixkyZAh33XUX\nJ588ibPP+zQ1NdtYsnYzu1rcJeuhgI+S3BDBgI+g38dbb73VFcCTJk1i0qRJXc//xBNPMG3aNKZO\nncrKlStZtWpVr+15++23+dznPkdeXh75+flceeWV/PWvfwXSv/2zMSYzfTJH/L2MzA8S7XBXPIby\nofR4N03V3YclrwwGDU/7qa6++mqefPJJdu7cybXXXsujjz5KXV0dj81/Hb8/yMwzTmZInp+xwwoR\ncaN8WrLp6/yfTZs28ZOf/ISFCxdSXFzMjTfe2ONtndN1wO2f/X4r9RhjDuChEX8KgWx3AU+4xQU+\nJG6oFj+0M3pw5Z558+bx5JNPcvXVV9PQ0EB2YQn4Amxe/h5bt2whNyuAL8Wpnueccw6PPfYYACtW\nrGDZsmUANDU1kZeXx6BBg6itreX555/vWifV7aDPPvts/vSnP9HW1kZraytPP/00Z5999iH1xxiT\nmT6ZI/5DFcp3B3n3fSlDNHH3QP+hndEzYcIEmpubGT58OEOGDGHGZz/Pr377ea698ExOm34qY8f2\nfl+Zb3zjG9x0002MGzeOcePGccop7j7tkydPZurUqYwdO5YRI0Zw5pn7v2tp9uzZzJw5k2HDhvH6\n6693TZ82bRo33ngj06dPB+Dmm29m6tSpVtYxxvQprXv1iMhM4D7cl6Y/rKr3dJtfDDwCnAB0AF9W\n1RWJed8GbsZ9A8Fy4CZV7bWO0e/36ol2ulutFlZCfrnbCDRscbeiPYTTOTujMeqaOumIxOiMxomr\nUp6fxdBP6FfLfVK/c9cYc7BDuVdPn6UeEfEDDwKzgPHA9SIyvttidwFLVXUS8EXcRgIRGQ7cDlSr\n6kTchiON+9L2M3/Inemz7773+y7eOoTb/e67yraxPYLfJ5TkhRhRksuQQUf3CxOMMaa/pVPqmQ5s\nUNWNACIyD7gcSD7tZDxwD4CqrhGRKhHZ9xVVASBHRCJALrC9vxqfNhF335zO5sQ3UYXdxiDNs3rC\n0Tgb61uJq3JCeR45vVxwZYwxx7p0km84sDXpcU1iWrIPgCsBRGQ6MBKoVNVtwE+ALcAOoFFVX+rp\nRURktogsEpFF9fX1PTbkiG4hHcpzN2GLdbp/aZZ43E3UWojFlVFl3gn9Y/F23MaYj0d/ndVzD1Ak\nIkuB24D3gVii9n85MAoYBuSJSI9XEqnqXFWtVtXq8vLyg+ZnZ2eze/fuww+skLt3DuFWV+pJ48Bu\nS0eUDXUtRGJKVVler7dW+CRRVXbv3k12tpWpjMlE6STZNmBE0uPKxLQuqtoE3AQg7n7Am4CNwEXA\nJlWtT8x7CvgU8LtDbWhlZSU1NTWk2hvokyo07YbtiW/dyumArJaUizd3RGlqjxDwu3r+liZvnfma\nnZ1NZWXlQDfDGDMA0gn+hcBoERmFC/zrgC8kLyAiRUCbqoZxZ/C8papNIrIFOF1EcoF2YAZw4Ok6\naQoGg4waNepwVt3v8R/C+hddyee6x2Ds9IMWicbi3PnUcp5cXMOsiUP48VUnU5B9BN/5aowxx5g+\ng19VoyJyK/Ai7qycR1R1pYh8PTF/DjAO+I2IKLAS+Epi3gIReRJYAkRxJaC5R6Un6Rh5Bqz9i/u5\n5PiDZndGY3xr3lKeX7GT22eM5tsXjP5kf6GJMcb0IK2itarOB+Z3mzYn6ed3gJNSrHs3cPcRtLH/\nHPep/T8XVx0wqy0c5Wu/Xcxf1+/iny8dz1fOOsK9C2OMOUZ542hluoZOcnfjzC484EvPVZVvzVvK\n3zbs4sefn8Q1p47o5UmMMeaTLbOC3x+EE2ccdKfO3y3Ywkuravm/l4yz0DfGeF5mBT/AVY8c8HDt\nzmZ++Nwqzj2pnC+faeUdY4z3ZV7wJ92moSMS47bHl1CQHeQnV0/G57MDucYY78u84E9y7yvrWVfb\nwv98eTrlBUf3u3eNMeZY4a2rkg6BqvLM0m1cMK6Cc046+EphY4zxqowN/nW1LWxv7OCCcYMHuinG\nGPOxytjgf21NHQDnj7XgN8ZklowN/tfX1jF+aCEVhXajMmNMZsnI4G9sj7D4o72cP9Zq+8aYzJOR\nwf/X9fXE4sqnrcxjjMlAGRn8r6+ppyg3yJQRxQPdFGOM+dhlXPDH48qb6+o4Z3Q5frtgyxiTgTIu\n+Jdva2RXS9jKPMaYjJVxwf/amjpEsIu2jDEZK6OCf/WOJn7zzmZOHVlCSV5ooJtjjDEDImOCf0Nd\nMzc8vIDsgJ//vGbyQDfHGGMGTEYE/+ZdrXzhoQWICI999TRGlOQOdJOMMWbApBX8IjJTRNaKyAYR\nubOH+cXTE8GUAAASzklEQVQi8rSILBOR90RkYtK8IhF5UkTWiMhqETmjPzuQjn/8wwdEYnEevfk0\nji/P/7hf3hhjjil9Br+I+IEHgVnAeOB6ERnfbbG7gKWqOgn4InBf0rz7gBdUdSwwGVjdHw1PVzQW\nZ9m2Rq46pZIxQwo+zpc2xphjUjoj/unABlXdqKphYB5webdlxgOvAajqGqBKRCpEZBBwDvDLxLyw\nqjb0W+vTsHl3G+FonDFDCj/OlzXGmGNWOsE/HNia9LgmMS3ZB8CVACIyHRgJVAKjgHrgVyLyvog8\nLCJ5R9zqQ7B2ZzMAY220b4wxQP8d3L0HKBKRpcBtwPtADPcNX9OAX6jqVKAVOOgYAYCIzBaRRSKy\nqL6+vp+aBWt3NuETOHGw1faNMQbSC/5twIikx5WJaV1UtUlVb1LVKbgafzmwEbd3UKOqCxKLPonb\nEBxEVeeqarWqVpeX99/FVWt2NlNVlkd20N9vz2mMMZ9k6QT/QmC0iIwSkRBwHfBM8gKJM3f2XRF1\nM/BWYmOwE9gqImMS82YAq/qp7WlZs7PZyjzGGJOkzy9bV9WoiNwKvAj4gUdUdaWIfD0xfw4wDviN\niCiwEvhK0lPcBjya2DBsBG7q5z6k1NoZZcueNq46pfLjekljjDnm9Rn8AKo6H5jfbdqcpJ/fAU5K\nse5SoPoI2njY1tW6A7t2Gqcxxuzn6St37YweY4w5mKeDf83OZnJDfkYU2y0ajDFmH08H/9qdzYyu\nKMBnX7hijDFdPBv8qsra2mbGVliZxxhjknk2+OtbOtnTGrYDu8YY041ng98O7BpjTM88H/w24jfG\nmAN5NvjX7GymLD+L0vysgW6KMcYcUzwb/GvtVg3GGNMjzwZ/XXMHw4tyBroZxhhzzPFs8EdiSijg\n2e4ZY8xh82wyRqJxgn7Pds8YYw6bZ5MxHIsTDNgVu8YY051ngz8SixOyEb8xxhzEk8kYjcWJK1bq\nMcaYHngyGSMxBSz4jTGmJ55MxnAsDkDQbzV+Y4zpzpPBH0kEv53OaYwxB/NkMka6Rvye7J4xxhyR\ntJJRRGaKyFoR2SAid/Ywv1hEnhaRZSLynohM7DbfLyLvi8hz/dXw3kSiVuM3xphU+kxGEfEDDwKz\ngPHA9SIyvttidwFLVXUS8EXgvm7z7wBWH3lz02M1fmOMSS2dIfF0YIOqblTVMDAPuLzbMuOB1wBU\ndQ1QJSIVACJSCVwCPNxvre5DV43fRvzGGHOQdJJxOLA16XFNYlqyD4ArAURkOjASqEzMuxf4JyDe\n24uIyGwRWSQii+rr69NoVmp2cNcYY1Lrr2S8BygSkaXAbcD7QExELgXqVHVxX0+gqnNVtVpVq8vL\ny4+oMXZw1xhjUgukscw2YETS48rEtC6q2gTcBCAiAmwCNgLXApeJyMVANlAoIr9T1Rv6oe0phe3g\nrjHGpJROMi4ERovIKBEJAdcBzyQvICJFiXkANwNvqWqTqn5PVStVtSqx3mtHO/QhudRjB3eNMaa7\nPkf8qhoVkVuBFwE/8IiqrhSRryfmzwHGAb8REQVWAl85im3uk5V6jDEmtXRKPajqfGB+t2lzkn5+\nBzipj+d4A3jjkFt4GCz4jTEmNU8mY9hu0maMMSl5MhkjUTuP3xhjUvFkMnaVeuzgrjHGHMSTwR+2\nGr8xxqTkyWQMRy34jTEmFU8m475v4LIavzHGHMyTyRixu3MaY0xKng1+EfD7LPiNMaY7TwZ/OBYn\n6PfhbhtkjDEmmSeDPxJVsqy+b4wxPfJkOkZicYJ2L35jjOmRJ9MxEovbgV1jjEnBk8G/r8ZvjDHm\nYJ5Mx0hM7Rx+Y4xJwZPpGInaiN8YY1LxZDq6g7tW4zfGmJ54Mvitxm+MMal5Mh0jFvzGGJNSWuko\nIjNFZK2IbBCRO3uYXywiT4vIMhF5T0QmJqaPEJHXRWSViKwUkTv6uwM9sYO7xhiTWp/pKCJ+4EFg\nFjAeuF5Exndb7C5gqapOAr4I3JeYHgW+o6rjgdOBW3pYt9/ZefzGGJNaOsPi6cAGVd2oqmFgHnB5\nt2XGA68BqOoaoEpEKlR1h6ouSUxvBlYDw/ut9SmE7aweY4xJKZ10HA5sTXpcw8Hh/QFwJYCITAdG\nApXJC4hIFTAVWHB4TU1f2G7ZYIwxKfVXOt4DFInIUuA24H0gtm+miOQDfwS+papNPT2BiMwWkUUi\nsqi+vv6IGhOJxa3Gb4wxKQTSWGYbMCLpcWViWpdEmN8EIO5eyJuAjYnHQVzoP6qqT6V6EVWdC8wF\nqK6u1vS7cLBIVK3Gb4wxKaQzLF4IjBaRUSISAq4DnkleQESKEvMAbgbeUtWmxEbgl8BqVf1pfza8\nN5FYnJCVeowxpkd9jvhVNSoitwIvAn7gEVVdKSJfT8yfA4wDfiMiCqwEvpJY/UzgH4DliTIQwF2q\nOr+f+3EAu4DLGGNSS6fUQyKo53ebNifp53eAk3pY723gY6+5WI3fGGNS82Q6RmJqI35jjEnBc+kY\niyuxuAW/Mcak4rl0jMTiAHZ3TmOMScGzwW81fmOM6Znn0jESc5cAWKnHGGN65rl07Cr1WPAbY0yP\nPJeO4ei+4LcavzHG9MRzwd9V47crd40xpkeeS0er8RtjTO88l45W4zfGmN55Lh3DMavxG2NMbzwX\n/JGoncdvjDG98Vw6hu3grjHG9Mpz6Wg1fmOM6Z3n0jEctbN6jDGmN55Lx/3n8dvBXWOM6Ylng99G\n/MYY0zPPpaMFvzHG9M5z6Ri2K3eNMaZXaaWjiMwUkbUiskFE7uxhfrGIPC0iy0TkPRGZmO66/c3O\n4zfGmN71mY4i4gceBGYB44HrRWR8t8XuApaq6iTgi8B9h7Buv7Jv4DLGmN6lMyyeDmxQ1Y2qGgbm\nAZd3W2Y88BqAqq4BqkSkIs11+5XV+I0xpnfppONwYGvS45rEtGQfAFcCiMh0YCRQmea6JNabLSKL\nRGRRfX19eq3vwb4af8BnI35jjOlJfw2L7wGKRGQpcBvwPhA7lCdQ1bmqWq2q1eXl5YfdkEgsTsjv\nQ8SC3xhjehJIY5ltwIikx5WJaV1UtQm4CUBc4m4CNgI5fa3b3yLRuN2Z0xhjepHOiH8hMFpERolI\nCLgOeCZ5AREpSswDuBl4K7Ex6HPd/haJxQnaDdqMMSalPkf8qhoVkVuBFwE/8IiqrhSRryfmzwHG\nAb8REQVWAl/pbd2j0xUnHFM7sGuMMb1Ip9SDqs4H5nebNifp53eAk9Jd92jaV+M3xhjTM88lZCQW\nt3vxG2NMLzyXkGE7uGuMMb3yXPBHYnGr8RtjTC88l5B2cNcYY3rnuYSMRO3grjHG9MZzCenO47ca\nvzHGpOLN4LcRvzHGpOS5hLQavzHG9M5zCWkXcBljTO88l5Cu1GM1fmOMScV7wR+1Gr8xxvTGcwkZ\njqndndMYY3rhuYS0Gr8xxvTOcwlpNX5jjOmdJ4Pf7s5pjDGpeSohVZWIncdvjDG98lRCRmIKYMFv\njDG98FRCRmJxADu4a4wxvUgrIUVkpoisFZENInJnD/MHicizIvKBiKwUkZuS5n07MW2FiDwuItn9\n2YFk+4LfDu4aY0xqfQa/iPiBB4FZwHjgehEZ322xW4BVqjoZOA/4TxEJichw4HagWlUn4r5w/bp+\nbP8BwtFE8NvBXWOMSSmdhJwObFDVjaoaBuYBl3dbRoECEREgH9gDRBPzAkCOiASAXGB7v7S8B+Gu\nEb8FvzHGpJJOQg4HtiY9rklMS/YAMA4X6suBO1Q1rqrbgJ8AW4AdQKOqvnTErU5h38Fdq/EbY0xq\n/ZWQFwFLgWHAFOABESkUkWLc3sGoxLw8EbmhpycQkdkiskhEFtXX1x9WIyI24jfGmD6lk5DbgBFJ\njysT05LdBDylzgZgEzAWuADYpKr1qhoBngI+1dOLqOpcVa1W1ery8vJD7QeQVOO3g7vGGJNSOsG/\nEBgtIqNEJIQ7OPtMt2W2ADMARKQCGANsTEw/XURyE/X/GcDq/mp8d10jfju4a4wxKQX6WkBVoyJy\nK/Ai7qycR1R1pYh8PTF/DvBvwK9FZDkgwHdVdRewS0SeBJbgDva+D8w9Ol2xGr8xxqSjz+AHUNX5\nwPxu0+Yk/bwduDDFuncDdx9BG9NmNX5jjOmbpxIybBdwGWNMnzwV/JGojfiNMaYvnkrIfTX+LDu4\na4wxKXkqIa3Gb4wxffNUQobtdE5jjOmTpxLS7s5pjDF981bwR+1+/MYY0xdPJaR9A5cxxvTNUwlp\nt2U2xpi+eSohrcZvjDF981Twh6Nxgn7B3Q/OGGNMTzwV/JFY3Mo8xhjTB0+lZCSmFvzGGNMHT6Vk\n2Eb8xhjTJ0+lZCQaJ2QHdo0xplfeCv5Y3G7XYIwxffBUSlqN3xhj+uaplAzH4na7BmOM6YOnUtJK\nPcYY07e0UlJEZorIWhHZICJ39jB/kIg8KyIfiMhKEbkpaV6RiDwpImtEZLWInNGfHUgWidnBXWOM\n6UufwS8ifuBBYBYwHrheRMZ3W+wWYJWqTgbOA/5TREKJefcBL6jqWGAysLqf2n6QSNRq/MYY05d0\nUnI6sEFVN6pqGJgHXN5tGQUKxN0rIR/YA0RFZBBwDvBLAFUNq2pDv7W+GzuP3xhj+pZOSg4HtiY9\nrklMS/YAMA7YDiwH7lDVODAKqAd+JSLvi8jDIpLX04uIyGwRWSQii+rr6w+1H4DdssEYY9LRXyl5\nEbAUGAZMAR4QkUIgAEwDfqGqU4FW4KBjBACqOldVq1W1ury8/LAaEYnFCQWsxm+MMb1JJ/i3ASOS\nHlcmpiW7CXhKnQ3AJmAsbu+gRlUXJJZ7ErchOCrsPH5jjOlbOim5EBgtIqMSB2yvA57ptswWYAaA\niFQAY4CNqroT2CoiYxLLzQBW9UvLe+Buy2zBb4wxvQn0tYCqRkXkVuBFwA88oqorReTriflzgH8D\nfi0iywEBvququxJPcRvwaGKjsRG3d3BUWI3fGGP61mfwA6jqfGB+t2lzkn7eDlyYYt2lQPURtDFt\nYTuP3xhj+uSp4XHESj3GGNMnT6XkhROGMGF44UA3wxhjjmlplXo+Kf7r2ikD3QRjjDnmeWrEb4wx\npm8W/MYYk2Es+I0xJsNY8BtjTIax4DfGmAxjwW+MMRnGgt8YYzKMBb8xxmQYUdWBbsNBRKQe+Ogw\nVy8DdvW5lLdkYp8hM/udiX2GzOz3ofZ5pKqm9WUmx2TwHwkRWaSqH8tN4Y4VmdhnyMx+Z2KfITP7\nfTT7bKUeY4zJMBb8xhiTYbwY/HMHugEDIBP7DJnZ70zsM2Rmv49anz1X4zfGGNM7L474jTHG9MIz\nwS8iM0VkrYhsEJE7B7o9R4uIjBCR10VklYisFJE7EtNLRORlEVmf+L94oNva30TELyLvi8hziceZ\n0OciEXlSRNaIyGoROcPr/RaRbyc+2ytE5HERyfZin0XkERGpE5EVSdNS9lNEvpfIt7UictGRvLYn\ngl9E/MCDwCxgPHC9iIwf2FYdNVHgO6o6HjgduCXR1zuBV1V1NPBq4rHX3AGsTnqcCX2+D3hBVccC\nk3H992y/RWQ4cDtQraoTAT9wHd7s86+Bmd2m9djPxN/4dcCExDo/T+TeYfFE8APTgQ2qulFVw8A8\n4PIBbtNRoao7VHVJ4udmXBAMx/X3N4nFfgNcMTAtPDpEpBK4BHg4abLX+zwIOAf4JYCqhlW1AY/3\nG/fNgDkiEgByge14sM+q+hawp9vkVP28HJinqp2qugnYgMu9w+KV4B8ObE16XJOY5mkiUgVMBRYA\nFaq6IzFrJ1AxQM06Wu4F/gmIJ03zep9HAfXArxIlrodFJA8P91tVtwE/AbYAO4BGVX0JD/e5m1T9\n7NeM80rwZxwRyQf+CHxLVZuS56k7Vcszp2uJyKVAnaouTrWM1/qcEACmAb9Q1alAK91KHF7rd6Km\nfTluozcMyBORG5KX8VqfUzma/fRK8G8DRiQ9rkxM8yQRCeJC/1FVfSoxuVZEhibmDwXqBqp9R8GZ\nwGUishlXxvu0iPwOb/cZ3KiuRlUXJB4/idsQeLnfFwCbVLVeVSPAU8Cn8Hafk6XqZ79mnFeCfyEw\nWkRGiUgIdxDkmQFu01EhIoKr+a5W1Z8mzXoG+FLi5y8Bf/6423a0qOr3VLVSVatwv9vXVPUGPNxn\nAFXdCWwVkTGJSTOAVXi731uA00UkN/FZn4E7juXlPidL1c9ngOtEJEtERgGjgfcO+1VU1RP/gIuB\ndcCHwPcHuj1HsZ9n4Xb/lgFLE/8uBkpxZwGsB14BSga6rUep/+cBzyV+9nyfgSnAosTv+09Asdf7\nDfw/YA2wAvgtkOXFPgOP445jRHB7d1/prZ/A9xP5thaYdSSvbVfuGmNMhvFKqccYY0yaLPiNMSbD\nWPAbY0yGseA3xpgMY8FvjDEZxoLfGGMyjAW/McZkGAt+Y4zJMP8/aAqR4i/kKJ4AAAAASUVORK5C\nYII=\n","text/plain":["<matplotlib.figure.Figure at 0x25224d73b00>"]},"metadata":{"tags":[]}}]},{"cell_type":"markdown","metadata":{"id":"qEbx6gFZ6E5J"},"source":["Stable and comparable result is obtained"]},{"cell_type":"code","metadata":{"id":"-fPvXbF16E5J","executionInfo":{"status":"ok","timestamp":1604779644789,"user_tz":420,"elapsed":3551377,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"901c2c9d-279f-431b-ff23-952edd089cb9","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":30,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 17s 56ms/step - loss: 0.0264 - accuracy: 0.9914\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"dW8aHTwP6E5M","executionInfo":{"status":"ok","timestamp":1604779644792,"user_tz":420,"elapsed":3551373,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"9270fd32-678e-4efd-8fc0-559b945d8872","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":31,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9914000034332275\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"-Pom-k3r6E5O"},"source":["## 3. Deep CNN - 3\n","- One of the ways to reduce the number of parameters in a model is proposed in Min et al 2013 (https://arxiv.org/abs/1312.4400)\n","- Namely \"Network In Network (NIN)\" or \"1 X 1 convolutions\"\n","    - Method is simple - adding convolutional layers after other convolutional layers\n","    - Has the effect of reducing the dimensionality (depth) of image space, effectively reducing the number of parameters\n","    \n","<img src=\"https://raw.githubusercontent.com/iamaaditya/iamaaditya.github.io/master/images/conv_arithmetic/full_padding_no_strides_transposed_small.gif\" style=\"width: 200px\"/>\n","\n","<br>\n","- GoogleNet architecture (Inception) utilizes NIN structure\n","\n","<img src=\"https://raw.githubusercontent.com/iamaaditya/iamaaditya.github.io/master/images/inception_1x1.png\" style=\"width: 800px\"/>"]},{"cell_type":"code","metadata":{"id":"9SdkH0Xe6E5O","executionInfo":{"status":"ok","timestamp":1604779644793,"user_tz":420,"elapsed":3551369,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["def deep_cnn_advanced_nin():\n","    model = Sequential()\n","    \n","    model.add(Conv2D(input_shape = (X_train.shape[1], X_train.shape[2], X_train.shape[3]), filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(Conv2D(filters = 25, kernel_size = (1,1), strides = (1,1), padding = 'valid', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(Conv2D(filters = 25, kernel_size = (1,1), strides = (1,1), padding = 'valid', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(Conv2D(filters = 25, kernel_size = (1,1), strides = (1,1), padding = 'valid', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(Conv2D(filters = 25, kernel_size = (1,1), strides = (1,1), padding = 'valid', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(Conv2D(filters = 50, kernel_size = (3,3), strides = (1,1), padding = 'same', kernel_initializer='he_normal'))\n","    model.add(Conv2D(filters = 25, kernel_size = (1,1), strides = (1,1), padding = 'valid', kernel_initializer='he_normal'))\n","    model.add(BatchNormalization())\n","    model.add(Activation('relu'))\n","    model.add(MaxPooling2D(pool_size = (2,2)))\n","    \n","    # prior layer should be flattend to be connected to dense layers\n","    model.add(Flatten())\n","    # dense layer with 50 neurons\n","    model.add(Dense(50, activation = 'relu', kernel_initializer='he_normal'))\n","    model.add(Dropout(0.5))\n","    # final layer with 10 neurons to classify the instances\n","    model.add(Dense(10, activation = 'softmax', kernel_initializer='he_normal'))\n","    \n","    adam = optimizers.Adam(lr = 0.001)\n","    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n","    \n","    return model "],"execution_count":32,"outputs":[]},{"cell_type":"code","metadata":{"id":"bwECqMpn6E5R","executionInfo":{"status":"ok","timestamp":1604779644981,"user_tz":420,"elapsed":3551554,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"source":["model = deep_cnn_advanced_nin()"],"execution_count":33,"outputs":[]},{"cell_type":"code","metadata":{"scrolled":true,"id":"NoseG0VU6E5U","executionInfo":{"status":"ok","timestamp":1604779644982,"user_tz":420,"elapsed":3551548,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"66ed1412-9980-4719-eb62-1bedef17e380","colab":{"base_uri":"https://localhost:8080/"}},"source":["model.summary()    # note that the number of parameters are greatly reduced"],"execution_count":34,"outputs":[{"output_type":"stream","text":["Model: \"sequential_3\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","conv2d_13 (Conv2D)           (None, 28, 28, 50)        500       \n","_________________________________________________________________\n","batch_normalization_6 (Batch (None, 28, 28, 50)        200       \n","_________________________________________________________________\n","activation_13 (Activation)   (None, 28, 28, 50)        0         \n","_________________________________________________________________\n","conv2d_14 (Conv2D)           (None, 28, 28, 50)        22550     \n","_________________________________________________________________\n","conv2d_15 (Conv2D)           (None, 28, 28, 25)        1275      \n","_________________________________________________________________\n","batch_normalization_7 (Batch (None, 28, 28, 25)        100       \n","_________________________________________________________________\n","activation_14 (Activation)   (None, 28, 28, 25)        0         \n","_________________________________________________________________\n","max_pooling2d_7 (MaxPooling2 (None, 14, 14, 25)        0         \n","_________________________________________________________________\n","conv2d_16 (Conv2D)           (None, 14, 14, 50)        11300     \n","_________________________________________________________________\n","conv2d_17 (Conv2D)           (None, 14, 14, 25)        1275      \n","_________________________________________________________________\n","batch_normalization_8 (Batch (None, 14, 14, 25)        100       \n","_________________________________________________________________\n","activation_15 (Activation)   (None, 14, 14, 25)        0         \n","_________________________________________________________________\n","conv2d_18 (Conv2D)           (None, 14, 14, 50)        11300     \n","_________________________________________________________________\n","conv2d_19 (Conv2D)           (None, 14, 14, 25)        1275      \n","_________________________________________________________________\n","batch_normalization_9 (Batch (None, 14, 14, 25)        100       \n","_________________________________________________________________\n","activation_16 (Activation)   (None, 14, 14, 25)        0         \n","_________________________________________________________________\n","max_pooling2d_8 (MaxPooling2 (None, 7, 7, 25)          0         \n","_________________________________________________________________\n","conv2d_20 (Conv2D)           (None, 7, 7, 50)          11300     \n","_________________________________________________________________\n","conv2d_21 (Conv2D)           (None, 7, 7, 25)          1275      \n","_________________________________________________________________\n","batch_normalization_10 (Batc (None, 7, 7, 25)          100       \n","_________________________________________________________________\n","activation_17 (Activation)   (None, 7, 7, 25)          0         \n","_________________________________________________________________\n","conv2d_22 (Conv2D)           (None, 7, 7, 50)          11300     \n","_________________________________________________________________\n","conv2d_23 (Conv2D)           (None, 7, 7, 25)          1275      \n","_________________________________________________________________\n","batch_normalization_11 (Batc (None, 7, 7, 25)          100       \n","_________________________________________________________________\n","activation_18 (Activation)   (None, 7, 7, 25)          0         \n","_________________________________________________________________\n","max_pooling2d_9 (MaxPooling2 (None, 3, 3, 25)          0         \n","_________________________________________________________________\n","flatten_3 (Flatten)          (None, 225)               0         \n","_________________________________________________________________\n","dense_6 (Dense)              (None, 50)                11300     \n","_________________________________________________________________\n","dropout_1 (Dropout)          (None, 50)                0         \n","_________________________________________________________________\n","dense_7 (Dense)              (None, 10)                510       \n","=================================================================\n","Total params: 87,135\n","Trainable params: 86,785\n","Non-trainable params: 350\n","_________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"xv8t7n_y6E5W","executionInfo":{"status":"ok","timestamp":1604782767616,"user_tz":420,"elapsed":6674175,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"f9e6a721-05bf-4c82-cb32-a9496c2dee22","colab":{"base_uri":"https://localhost:8080/"}},"source":["%%time\n","history = model.fit(X_train, y_train, batch_size = 50, validation_split = 0.2, epochs = 10, verbose = 1)"],"execution_count":35,"outputs":[{"output_type":"stream","text":["Epoch 1/10\n","960/960 [==============================] - 317s 330ms/step - loss: 0.4332 - accuracy: 0.8612 - val_loss: 0.0894 - val_accuracy: 0.9717\n","Epoch 2/10\n","960/960 [==============================] - 312s 325ms/step - loss: 0.1419 - accuracy: 0.9578 - val_loss: 0.1064 - val_accuracy: 0.9716\n","Epoch 3/10\n","960/960 [==============================] - 316s 329ms/step - loss: 0.1058 - accuracy: 0.9690 - val_loss: 0.0532 - val_accuracy: 0.9843\n","Epoch 4/10\n","960/960 [==============================] - 309s 322ms/step - loss: 0.0871 - accuracy: 0.9745 - val_loss: 0.0701 - val_accuracy: 0.9787\n","Epoch 5/10\n","960/960 [==============================] - 313s 327ms/step - loss: 0.0764 - accuracy: 0.9771 - val_loss: 0.0467 - val_accuracy: 0.9876\n","Epoch 6/10\n","960/960 [==============================] - 307s 319ms/step - loss: 0.0723 - accuracy: 0.9785 - val_loss: 0.0367 - val_accuracy: 0.9898\n","Epoch 7/10\n","960/960 [==============================] - 308s 321ms/step - loss: 0.0635 - accuracy: 0.9808 - val_loss: 0.0796 - val_accuracy: 0.9791\n","Epoch 8/10\n","960/960 [==============================] - 309s 322ms/step - loss: 0.0581 - accuracy: 0.9824 - val_loss: 0.0510 - val_accuracy: 0.9881\n","Epoch 9/10\n","960/960 [==============================] - 311s 323ms/step - loss: 0.0535 - accuracy: 0.9837 - val_loss: 0.0557 - val_accuracy: 0.9869\n","Epoch 10/10\n","960/960 [==============================] - 317s 330ms/step - loss: 0.0466 - accuracy: 0.9858 - val_loss: 0.0459 - val_accuracy: 0.9878\n","CPU times: user 1h 33min 28s, sys: 3min 14s, total: 1h 36min 43s\n","Wall time: 52min 2s\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"R0U4HGfW6E5Y","executionInfo":{"status":"ok","timestamp":1604782768539,"user_tz":420,"elapsed":887,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"05da7988-06bd-4dcc-91d5-5b5feb82a9bc","colab":{"base_uri":"https://localhost:8080/","height":265}},"source":["plt.plot(history.history['accuracy'])\n","plt.plot(history.history['val_accuracy'])\n","plt.legend(['training', 'validation'], loc = 'upper left')\n","plt.show()"],"execution_count":36,"outputs":[{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3yU5Zn/8c+VZCaTc0IIBBJOKgqIyCEilApUrcVDdaUHtbUt9kB/VmvtYbu4uy/buuu2v13Wtf3Zw6rFtrbWpbRubUtr1WLRFi0JAoLIQWFCEoFAJidymplcvz+eh2QIgQwyYU7X+/XKK888h8k1o3znnvu5n/sRVcUYY0zqyoh3AcYYY4aXBb0xxqQ4C3pjjElxFvTGGJPiLOiNMSbFZcW7gIFGjhypEydOjHcZxhiTVGpqag6ratlg2xIu6CdOnEh1dXW8yzDGmKQiIv6TbbOuG2OMSXEW9MYYk+Is6I0xJsUlXB/9YILBIHV1dXR1dcW7lJTh8/morKzE4/HEuxRjzDBLiqCvq6ujoKCAiRMnIiLxLifpqSpHjhyhrq6OSZMmxbscY8wwS4qum66uLkpLSy3kY0REKC0ttW9IxqSJpAh6wEI+xuz9NCZ9JEXXjTFJqTcMRw9D+wFoc386DsOEBTB+XryrM2nEgj5Kzc3NPPHEE3zuc587reOuueYannjiCYqLi0+6z7333svChQu58sorz7RMczaEg9B+yA3wg8cHefvB/uWjjaDhwZ9j4mWw8O9h0kKwb1dmmEmi3XikqqpKB14Zu2PHDqZOnRqnihz79u3juuuuY9u2bcetD4VCZGUl5+dlIryvCSXU7Qb1QWh7uz+0+4LcDfWjh4GB/24E8kZCfjkUlEPB6P7l/NFQMMZZl10Am38Of/m281yVc53An/xeC3xzRkSkRlWrBtuWnAkVBytWrODNN99k5syZeDwefD4fJSUlvPHGG+zatYu/+7u/Y//+/XR1dfGFL3yB5cuXA/1TOrS3t3P11Vfz7ne/m7/+9a9UVFTw61//mpycHJYtW8Z1113HBz/4QSZOnMgnPvEJfvOb3xAMBvnFL37BlClTaGxs5CMf+QgNDQ3Mnz+fZ599lpqaGkaOHBnndyZJdDRB487+VvhxQe4+7gyceJxkQv4oJ6yLKqBidn9o57uBXjAG8sogM8qhqvM/B1WfhM0/g5cehCc+BOUznMCfch1kJM2ps3dGFfa/Aht/CG8+D1k+8OY7H4LZBZCdD9mFEeuO/R64rtBZ9uaDJyexPihVIdgJwQ7oOeouH4WejgHL7s+x5YJymH9HzMtJuqD/xm+283pDa0yfc9rYQr72/gtPuc+3vvUttm3bxubNm3nhhRe49tpr2bZtW9/wxFWrVjFixAg6Ozu55JJL+MAHPkBpaelxz7F7925+/vOf88gjj/DhD3+YX/7yl9x6660n/K2RI0eyadMmvve977Fy5UoeffRRvvGNb3D55Zdzzz338Ic//IEf/vCHsXsDUpUq7F0PNY/Bjt9Cb7B/W4anv7U94hwYP//EAM8vd1rpGZmxr83jg0s+BbM/Dlv/B178T1j9MSibCgu/AhfeODx/N566Wp3XWv0YHNruBPWUa50P0+5W6GmH7jbnw7e7DXranN/aO/RzS+aJ4R/5IeEtOMUHSQFkep3wDXa6oXts+agbxlEsRwZ3sOP0358sH4yba0GfSObOnXvcGPTvfOc7PPXUUwDs37+f3bt3nxD0kyZNYubMmQDMmTOHffv2DfrcS5cu7dvnV7/6FQAvvfRS3/MvWbKEkpKSmL6elHL0iNNarvkRNL0JvmK45NMw+UonzPPLIackMVrOmR6YdSvMuBle/19YvxJ++SlY929w2Zdgxk3Rf1NIVAdec1rvr/3CCfPyGfD+b8P0DzqheyqqTmh2tx8f/t1t7rqID4hj+xxb19UMLXXuce62E7rcTkOWDzy5zo83t385dwR4Kk9c780FT57zbeO45Tx3nwHLw/jBnnRBP1TL+2zJy8vrW37hhRd47rnn2LBhA7m5uSxevHjQMerZ2dl9y5mZmXR2dg763Mf2y8zMJBQKxbjyFKUK+15yW++/gXAPjJsHi74K025w/iElsswsuOiDcOFSeOO3sP4/4Nd3wAv/F959N8z8qPMtIFkEu2D7U1C9Cur+5oTk9A9A1aec7q9ou1lEnDD05jnfss5Eb6/TUh/sQyPcc9aDuLdXCXT0cKipm8a2Jg61dZPjyeTaGWNi+ncgCYM+XgoKCmhraxt0W0tLCyUlJeTm5vLGG2/w8ssvx/zvL1iwgNWrV/MP//AP/PGPfyQQGKQ/OR11NMHmJ5zW+5HdkF0Ec26DOctg9LR4V3f6MjJg2vUw9f2w+1lY/+/wuy85wf+uu5zX5c2Nd5Und+RNJ9w3/8w551F6Hrzv3+DiW5yWbzxlZPR31RD7MD2mJ9TL4fZuDrV1c6i1i0Nt3TS2dbu/+x83tnUT6j3+G8b0ikIL+ngqLS1lwYIFTJ8+nZycHEaP7m9dLFmyhB/84AdMnTqVCy64gHnzYj9G+mtf+xq33HILjz/+OPPnz6e8vJyCgoKTH6Dq9ElneBLrJFUsqELtBqev9/VfQ7jbGb1yw/ecvu1EDsJoicD5Vzmjcfaud4L+mXucvvz5dzhdUb7CeFfpCIdg51on4N9aBxlZTt971Sdh0qKU+f+vvTt0QnAfauuisbWbxvZuDrU6jwMdwROOFYHSPC8j87MZVejj/NEFjCrIpqwgm1EFPkYVZvc9Hg42vDJJdHd3k5mZSVZWFhs2bOD2229n8+bNJ9m5DVrqIdTpBH12fv+Jp0xv3z+8pHtfOwOw5Umn9d74hnNCbcZNTiu3fHq8qxt+/g3w4krY85xz3mHe7XDpZ53zDfHQ2gA1P4ZNP3ZGLRVWOP8tZn/cOdGd4Hp7laM9Idq7QzR3BE9ogTe6QX7scUfPiddEeDMzKOsL7MGDe1SBj9J8L57M4T0nZMMrU0BtbS0f/vCH6e3txev18sgjj5y4U6jb+cfX1ewEesEYp6+0u61/6GCGp3/kQW8S9P+rwv6/OX3v25+CUBdUVMH1D8H0pU4farqYMB8m/BLqNzknbV/4Jvz1IZj7aZh/pzNCaLj19jqt9upVsPP3zoiY866Aa/8TJr/POddwFnQFw7R3h2jrCtHeFaKtK0hrV8hdF3TWudvbuoIn7NvWFaK9J8TJ2rkF2VmUFWZTlp/NjMpiRp0kyItyPEkxnYi16FNBb9i9UvMgIM5Jq7xR/aNKVJ0PgR53ZEJPO/SG2OE/xNRX/h4mXQYTFzq/C8fG9aX06Wx2huLV/AgOve4Mj5vxYafFOGZGvKtLDAe2OV05259yTnZWfRLe9XkoHIb+544mePWnzgdu01uQW+qMFppzG4w4vRlQQ+FemjuDNHf0EOgI9gVv28Cg7gvrYMQ2J6x7wkMPufR5MsjP9lDoyyLfl0WBL4v87CwKfB7ys7Mi1nsoyvEcF+Q53uQb2mot+lSl6rTUWxuc/nhfiRPUWd7j9xNxRmx4fM6FPapOy/hQEMovckapvPpTZ98R57rBf5lzeX7+qLP7euqqnTDZ9iun62nsLHj/d5wRG0MNxUs35dPhQ4/Be/4RXnwAXvkBbHwEZn3MGalTPP7Mnl8V6jY6QyO3P+WcCxk/Hxb/o3PCOCub7lCY5tYuAh09NB3tobkj6P52QjxwtIfAseWOHgJHe2jtOvU3ycwMcQM5yw1kD+WFvoiw9lDgLkc+PrZvgS+LvOwsvFkJMHw2QUTVoheRJcC3gUzgUVX91oDtE4BVQBnQBNyqqnXutn8HrsWZKfNZ4At6ij9qLfoo9XQ4Y4SDRyErB4oqTzsI+97X3rAz1nnfi7D3RfD/1Wn9A4y8oD/4J14GeaWnftJ3oqsFtq52Wu8HtznnEy76oNNaHDsz9n8vVTXthb88CK/+DFBnbP5lX4LSc4c8VFXpDIYJdARpCTTheX0No3Y9QVHLG3Rn5rGl5H08n38dr4crjwvzo4P0Wx+T582kONdLSZ6HklwvJbleRuR5Kc51H+d5Kc7xUJhzfAs7x5OZFN0hieZULfohg15EMoFdwHuBOmAjcIuqvh6xzy+A36rqj0XkcuA2Vf2YiLwL+A9gobvrS8A9qvrCyf6eBf0QwkHnxFfHEWd0Q8EY52v0O/iHcdL3NRyCt7fAvvVO8Ne+7HygAIye7rb2L4MJ73rnJwJVnb7mmsdg2y+di2LGXOyE+0UfdIfAmXekpR7++h2o+REa7qH9vBvYM2U5/owJHGzt4mBrNwfbutzWdn+re1J4Hx/NfI4bM18iX7p4vXcCPw1fya/D7yLDV+CGtJcRblAX53oZkedxf/cH+LHl7Kzk6/5IZmfadTMX2KOqb7lP9iRwA/B6xD7TgC+5y+uA/3WXFfABXkAAD3DwdF+AwTnpdbTRmadFe50++ILRTtjHWmYWVM5xft79RefDpX5Tf/DXPAavfB8Qp7/8WDfP+PlDD/nrbnNb74853yI8ef2t94rZsX8tKao7FOZQa3d/cLd2cbCti0Ot3Rxo6eJg29WEg7O5Jfw0H9u1llm7n+JA+BKeDv0dez3nMbrQx4g8LxMKM/h47ibe3fJrxrW/RjjDy4Fx1+C/8ON4x1/Cl/Kz+UaOZ9hHjJjhFU1KVAD7Ix7XAZcO2GcLsBSne+dGoEBESlV1g4isA97GCfqHVHXHwD8gIsuB5QDjx59hv2KCyM/Pp729nYaGBu666y7WrFlzwj6LFy9m5cqVVFUN+iEMwIMPPsjyj99EbjAA4W6u+cQXeeKJJykuOovD1zI9MP5S52fh3zsnduuq+7t6/vYwbHjImW9k7Mz+Fv/4+f2jYhpedca9v7bG+XZQfhFc+wBc9KHEGQ+eAELhXo4c7eFga5cb2M6Qv4OtXRxo7V8ebKy2NzODUYXZjC70MaW8gFGTy6Coihd89zCj7udctfMnXN3zT+jk9yGXfBr2Pe+cm+lscub7uepfyZz5USpyR1ARh9duhk+smoNfAR4SkWXAeqAeCIvIecBUoNLd71kRuUxVX4w8WFUfBh4Gp+smRjUlhLFjxw4a8lEJdvHgAyu59coZ5JaVw4hzWPvsCzGt7x3JyoaJC5yfxSuciZ32/60/+Dd81+krzsiCijnOB8Pbm51LyacvhTmfPL3L4JNcb6/S1h2itTNIS2fQvbgmoiXe97uLw+3dDLhYkgyBsgInwCtLcpkzoYTyQh+jC319wT660EdJ7imG+l16P3R9Ff72MLLhe7D7GeeDeco1zrQEkxYlxtw/ZlhEE/T1wLiIx5Xuuj6q2oDTokdE8oEPqGqziHwGeFlV291tvwfmA8cFfTJYsWIF48aN4447nJnlvv71r5OVlcW6desIBAIEg0H+9V//lRtuuOG44yLnse/s7OS2225jy5YtTJky5bi5bm6//XY2btxIZ2cnH/zAUr7x5f/Dd77zbRoOHOQ9N93ByFHlrFu3rm/a45EjR/LAAw+watUqAD796U9z9913s2/fvpNOhzxsPDlwziLnB5zZ/Gpf7g9+EbhmpTM80lc0fHUME1WloydMa1eQ1s4QLZ1BWjuD7uMgLZ2hvuXWrqC7vX9dW/fJx2uPyPMyyg3xqWMKKC/0McoN7tFuiI/MzyYzIwYfir4i5xvZpbfDm3+CyqrEGU5rhlU0Qb8RmCwik3AC/mbgI5E7iMhIoElVe4F7cEbgANQCnxGRb+J03SwCHjyjin+/wunbjaXyi+Dqb51yl5tuuom77767L+hXr17NM888w1133UVhYSGHDx9m3rx5XH/99SdtVX3/+98nNzeXHTt2sHXrVmbP7u+Tvv/++xlRUkK47RBXLLmOrQunc9fn7+SBH65m3Z9fPGHe+ZqaGh577DFeeeUVVJVLL72URYsWUVJSEvV0yMPGm+dcRHPeFWfvbw4hGO4l0NHTF8D9Ye20tPuDe/DtA+ckGSjPm0lhjodCn4fCnCzGFvuY4itw12U5v93tZQXevotu4nLCMjvfGR5p0saQQa+qIRG5E3gGZ3jlKlXdLiL3AdWq+jSwGPimiChO182xCZXXAJcDr+GcmP2Dqv4m9i9j+M2aNYtDhw7R0NBAY2MjJSUllJeX88UvfpH169eTkZFBfX09Bw8epLx88P7z9evXc9dddwEwY8YMZszov/Bn9ROP8/DD/00oFOTtQ0d4/UAXM04xDvqll17ixhtv7JtFc+nSpbz44otcf/31UU+HnErauoI0NHdR39xBfaCT+uYu6ps7aWjupD7QycG2rpO2qgG8WRkURYRySa6XiaV5FOZkueHtcbd7+tYVueFd4Muyk5UmoUXVR6+qa4G1A9bdG7G8BifUBx4XBj57hjUeb4iW93D60Ic+xJo1azhw4AA33XQTP/vZz2hsbKSmpgaPx8PEiRMHnZ74lEJB9r76Iiv/cyUbf/8kJeOnsuxzX6Ir9M5PVUQ7HXKy6O1VDrd3UxcR3A3NndQ3d1LnLg+8CMeTKYwpyqGiOIcF542koiSHsnzvcS3ropysvmWfx4YCmtRlV8aehptuuonPfOYzHD58mD//+c+sXr2aUaNG4fF4WLduHX6//5THL1y4kCeeeILLL7+cbVu3snXrVgjspTUrk7z8QorOm8vBI038/ve/Z/HixUD/9MgDu24uu+wyli1bxooVK1BVnnrqKR5//PHheunDqjsU5m23Be60xjv7W+PNnbzd3HXCJe8Fviwqip0gnztpBGPd5bHFOVSW5FCWn01GLPq1jUkBFvSn4cILL6StrY2KigrGjBnDRz/6Ud7//vdz0UUXUVVVxZQpU055/O23385tt93G1CnnM/Xc8cyZMRW8eVy8aAmz5vyeKdMuZNy4cSxYsKDvmOXLl7NkyRLGjh3LunXr+tbPnj2bZcuWMXfuXMA5GTtr1qyE66ZRVVo7Q9Q1dzhdK4EOGlq6qA909rXQG9u6jztGBEYX+Bhb7GNGZTFLpvuoLM6hosQJ8rHFORT6kvyuS8acRTap2dkU7HCuWuxpdyahKqqM6xWgw/G+qir7mzp5+a0jfT8NLcd3Z2VnZTit8ZIcxhY5vyNb46MLfTZPiTGnySY1i7dwyJ224LAzdrmoEnJHpsQ48lMF+8h8L5eeU8ptlcVUlvS3yEvzvDaXiTFnkQX9cNJeOHrYuau9hp2ZI/PLz9qc3cMhmmC//ZxS5p8zgnPL8i3QjUkASZM4qppcodHVCq31znTA3gIoqkioG1RH22VnwW5M8kuKoPf5fBw5coTS0tKTB0lvCAK1/Y9P2G3gChlk9SDPLSdZf6p9Q+5dnTK9UDLJuSIxgQJQVTly5Ag+n2/QbScL9tI8L/POKeX2c0Yw75xSzhtlwW5MMkiKoK+srKSuro7GxsaT79TbC0cPDVg5SKt10IZsNK1bjf54EacVn50NTW/jzOmWWHw+H5WVlagqdYFONrih/spbTdQ3O+PuLdiNSQ1JEfQej4dJk6K5XdmFw15LsosM9v+3cfugwf5/Fp1jwW5MCkmKoDfvXDQtdgt2Y1KbBX2K+tveJp7cWHtcsI/I8zLvnBF81g32yRbsxqQFC/oUU+MP8F/P7uKlPYcpzvXwrnNLLdiNSXMW9Cliy/5m/uu5Xbyws5HSPC//fO1Ubp03wSbrMsZY0Ce7bfUtPPjcLp7bcYiSXA8rrp7Cx+dPINdr/2mNMQ5LgyT1xoFWHnx2N3/YfoBCXxZfuep8li2YRH62/Sc1xhzPUiHJ7D7YxoPP7+Z3W9+mIDuLu6+czCffPclmczTGnJQFfZJ4q7Gdbz+/m6e3NJDryeTO95zHpy+bRHGuN96lGWMSnAV9gvMfOcp3nt/DU6/WkZ2VyfKF5/DZhecyIs8C3hgTHQv6BFUX6OChP+3hFzV1ZGUIn1wwic8uOpeyguyhDzbGmAgW9Anm7ZZOHvrTHlZX70cQPjZvAp9bfC6jCk+cgMwYY6JhQZ8gDrV28b0X3uSJV2pRlJsuGccd7zmPMUWJM7WxMSY5WdDHWWNbNz/485v89GU/oV7lQ3MqueM95zFuRG68SzPGpIiogl5ElgDfBjKBR1X1WwO2TwBWAWVAE3Crqta528YDjwLjcObzvUZV98XqBSSrpqM9/Pf6N/nJX/10h8IsnV3J5y8/jwmlefEuzRiTYoYMehHJBL4LvBeoAzaKyNOq+nrEbiuBn6jqj0XkcuCbwMfcbT8B7lfVZ0UkH+iN6StIMs0dPTz64l4e+8teOoJhbrh4LHddMZlzyvLjXZoxJkVF06KfC+xR1bcARORJ4AYgMuinAV9yl9cB/+vuOw3IUtVnAVS1PUZ1J52WziCrXtrLqpf20tYd4toZY7j7islMHl0Q79KMMSkumqCvAPZHPK4DLh2wzxZgKU73zo1AgYiUAucDzSLyK2AS8BywQlXDkQeLyHJgOcD48ePfwctIXO3dIX70l708vP4tWrtCLLmwnLvfO5kp5YXxLs0YkyZidTL2K8BDIrIMWA/UA2H3+S8DZgG1wP8Ay4AfRh6sqg8DDwNUVVVFd9fqBNfRE+LHf/Xz8Po3CXQEuXLqaO6+cjLTK4riXZoxJs1EE/T1OCdSj6l01/VR1QacFj1uP/wHVLVZROqAzRHdPv8LzGNA0KcSVWXVX/bxvXV7OHK0h8UXlPHFK8/n4nHF8S7NGJOmogn6jcBkEZmEE/A3Ax+J3EFERgJNqtoL3IMzAufYscUiUqaqjcDlQHWsik9E63Ye4l9++zrvOreUL191AXMmlMS7JGNMmssYagdVDQF3As8AO4DVqrpdRO4Tkevd3RYDO0VkFzAauN89NozTrfO8iLwGCPBIzF9FAnllbxOeTGHVskss5I0xCSGqPnpVXQusHbDu3ojlNcCakxz7LDDjDGpMKpv8AaZXFNmdnYwxCWPIFr2JXncozJa6FuaMt5a8MSZxWNDH0PaGVnpCvdZlY4xJKBb0MbTJHwCwoDfGJBQL+hiq3hdg3Igcm1LYGJNQLOhjRFWpqQ1Y/7wxJuFY0MdIXaCTxrZu5kwcEe9SjDHmOBb0MVLtbwKwFr0xJuFY0MdIjT9AnjeTC8ptNkpjTGKxoI+RGn8zs8aXkJkh8S7FGGOOY0EfA21dQXYeaLVhlcaYhGRBHwOb9zfTqzZ+3hiTmCzoY6DGH0AEZo63qYiNMYnHgj4GavwBLhhdQKHPE+9SjDHmBBb0Zyjcq7xa22zdNsaYhGVBf4Z2HWyjvTtkQW+MSVgW9Geoxp3IrGqCXRFrjElMFvRnaJM/wMj8bMaNyIl3KcYYMygL+jNU7Q8wZ0IxInahlDEmMVnQn4FDbV3UNnVY/7wxJqFZ0J+BTf5mAOZY/7wxJoFZ0J+BGn8T3swMplcUxrsUY4w5KQv6M1DjD3BRZRHZWZnxLsUYY04qqqAXkSUislNE9ojIikG2TxCR50Vkq4i8ICKVA7YXikidiDwUq8LjrSsYZlt9K1XWP2+MSXBDBr2IZALfBa4GpgG3iMi0AbutBH6iqjOA+4BvDtj+L8D6My83cWyrb6En3MtsC3pjTIKLpkU/F9ijqm+pag/wJHDDgH2mAX9yl9dFbheROcBo4I9nXm7iOHah1Gy7o5QxJsFFE/QVwP6Ix3XuukhbgKXu8o1AgYiUikgG8J/AV071B0RkuYhUi0h1Y2NjdJXHWY0/wITSXMoKsuNdijHGnFKsTsZ+BVgkIq8Ci4B6IAx8DlirqnWnOlhVH1bVKlWtKisri1FJw0dV2VQbsPHzxpikkBXFPvXAuIjHle66PqragNuiF5F84AOq2iwi84HLRORzQD7gFZF2VT3hhG4y8R/p4HB7jwW9MSYpRBP0G4HJIjIJJ+BvBj4SuYOIjASaVLUXuAdYBaCqH43YZxlQlewhD/398xb0xphkMGTXjaqGgDuBZ4AdwGpV3S4i94nI9e5ui4GdIrIL58Tr/cNUb0KoqQ1QkJ3F+aMK4l2KMcYMKZoWPaq6Flg7YN29EctrgDVDPMePgB+ddoUJqGZfgFkTSsjIsInMjDGJz66MPU0tnUF2HWpjjg2rNMYkCQv607R5fzOqUDXRgt4Ykxws6E9Tzb4mMgQuHlcc71KMMSYqFvSnqaY2wJTyQvKzozq9YYwxcWdBfxpC4V421zbbsEpjTFKxoD8NOw+2cbQnbP3zxpikYkF/GmwiM2NMMrKgPw01/gCjCrKpLMmJdynGGBM1C/rTUOMPUDWxBBG7UMoYkzws6KN0sLWLukCnddsYY5KOBX2UbCIzY0yysqCPUo0/QHZWBheOLYp3KcYYc1os6KNU4w9wcWUx3ix7y4wxycVSKwpdwTDbG1rsRuDGmKRkQR+FrXUtBMNq/fPGmKRkQR8FOxFrjElmFvRRqPE3cc7IPEbkeeNdijHGnDYL+iGoKjX+gPXPG2OSlgX9EPYePkqgI0iVBb0xJklZ0A+h2vrnjTFJzoJ+CJv8AQp9WZxblh/vUowx5h2JKuhFZImI7BSRPSKyYpDtE0TkeRHZKiIviEilu36miGwQke3utpti/QKG27H++YwMm8jMGJOchgx6EckEvgtcDUwDbhGRaQN2Wwn8RFVnAPcB33TXdwAfV9ULgSXAgyKSNDdbbekIsvtQu/XPG2OSWjQt+rnAHlV9S1V7gCeBGwbsMw34k7u87th2Vd2lqrvd5QbgEFAWi8LPhk217o1GLOiNMUksmqCvAPZHPK5z10XaAix1l28ECkSkNHIHEZkLeIE331mpZ1+NP0BmhjBzXNJ8CTHGmBPE6mTsV4BFIvIqsAioB8LHNorIGOBx4DZV7R14sIgsF5FqEalubGyMUUlnrsYfYNqYQnK9WfEuxRhj3rFogr4eGBfxuNJd10dVG1R1qarOAv7JXdcMICKFwO+Af1LVlwf7A6r6sKpWqWpVWVli9OwEw71s3t9swyqNMUkvmqDfCEwWkUki4gVuBp6O3EFERorIsee6B1jlrvcCT+GcqF0Tu7KH3xtvt9EZDFv/vDEm6Q0Z9KoaAu4EngF2AKtVdbuI3Cci17u7LQZ2isguYDRwv7v+w8BCYJmIbHZ/Zsb6RQyHGn8TYBdKGWOSX1Sdz6q6Flg7YN29EctrgDAp7+YAAAyLSURBVBNa7Kr6U+CnZ1hjXNTUNjOmyEdFcU68SzHGmDNiV8aeRM2+Juu2McakBAv6QTQ0d9LQ0sWc8Rb0xpjkZ0E/iGMXSlVNtKA3xiQ/C/pBVO8L4PNkMHVMYbxLMcaYM2ZBP4hNtQEurizGk2lvjzEm+VmSDdDRE2J7Q6t12xhjUoYF/QBb9rcQ7lUbP2+MSRkW9AMcOxE7a5wFvTEmNVjQD1DjD3BuWR4led54l2KMMTFhQR+ht1fZVBugasKIeJdijDExY0Ef4a3D7TR3BK1/3hiTUizoI9T47Y5SxpjUY0EfocYfoDjXw7llefEuxRhjYsaCPkK1P8Cc8SWISLxLMcaYmLGgdzUd7eGtxqPWbWOMSTkW9K5Xj01kZkFvjEkxFvSuGn+ArAxhRmVxvEsxxpiYsqB3VfsDXDi2kBxvZrxLMcaYmLKgB4LhXrbsb7b+eWNMSrKgB15vaKU71GtXxBpjUpIFPU63DcDsCdY/b4xJPRb0wCZ/gIriHMYU5cS7FGOMibmogl5ElojIThHZIyIrBtk+QUSeF5GtIvKCiFRGbPuEiOx2fz4Ry+JjQVWp9jfZ/DbGmJQ1ZNCLSCbwXeBqYBpwi4hMG7DbSuAnqjoDuA/4pnvsCOBrwKXAXOBrIpJQiVrf3MnB1m4LemNMyoqmRT8X2KOqb6lqD/AkcMOAfaYBf3KX10Vsfx/wrKo2qWoAeBZYcuZlx86xicws6I0xqSqaoK8A9kc8rnPXRdoCLHWXbwQKRKQ0ymMRkeUiUi0i1Y2NjdHWHhOb/AFyvZlMKS84q3/XGGPOllidjP0KsEhEXgUWAfVAONqDVfVhVa1S1aqysrIYlRSdmtoAM8cVk5Vp56WNMakpmnSrB8ZFPK501/VR1QZVXaqqs4B/ctc1R3NsPB3tDrHj7TbrtjHGpLRogn4jMFlEJomIF7gZeDpyBxEZKSLHnuseYJW7/AxwlYiUuCdhr3LXJYQt+5sJ96pdEWuMSWlDBr2qhoA7cQJ6B7BaVbeLyH0icr2722Jgp4jsAkYD97vHNgH/gvNhsRG4z12XEPruKDXegt4Yk7qyotlJVdcCawesuzdieQ2w5iTHrqK/hZ9Qqv0Bzh+dT1GOJ96lGGPMsEnbM5C9vcqm2oD1zxtjUl7aBv2exnbaukLMsYnMjDEpLm2D3i6UMsaki7QN+up9AUbkeZlYmhvvUowxZlilbdBvqg0we3wJIhLvUowxZlilZdAfae9m7+GjVE20bhtjTOpLy6C3/nljTDpJz6CvDeDJFC6qKIp3KcYYM+zSMug3+QNMryjC58mMdynGGDPs0i7ou0NhttS1MMemPTDGpIm0C/rtDa30hHqtf94YkzbSLug32YlYY0yaSbugr/EHGDcih1GFvniXYowxZ0VaBb2qUu0PWP+8MSatpFXQ1wU6aWzrtm4bY0xaSaug779QymasNMakj7QK+mp/E3neTC4oL4h3KcYYc9akVdDX+JuZNb6EzAybyMwYkz7SJujbuoLsPNBq/fPGmLSTNkG/ZX8LvWrj540x6Sdtgr7a34QIzBxfHO9SjDHmrEqboK/xB7hgdAGFPk+8SzHGmLMqqqAXkSUislNE9ojIikG2jxeRdSLyqohsFZFr3PUeEfmxiLwmIjtE5J5Yv4BohHuVzbXN1m1jjElLQwa9iGQC3wWuBqYBt4jItAG7/TOwWlVnATcD33PXfwjIVtWLgDnAZ0VkYmxKj96ug220dYcs6I0xaSmaFv1cYI+qvqWqPcCTwA0D9lGg0F0uAhoi1ueJSBaQA/QArWdc9WmyO0oZY9JZNEFfAeyPeFznrov0deBWEakD1gKfd9evAY4CbwO1wEpVbRr4B0RkuYhUi0h1Y2Pj6b2CKGzyBxiZn834Ebkxf25jjEl0sToZewvwI1WtBK4BHheRDJxvA2FgLDAJ+LKInDPwYFV9WFWrVLWqrKwsRiX1q/YHmDOhGBG7UMoYk36iCfp6YFzE40p3XaRPAasBVHUD4ANGAh8B/qCqQVU9BPwFqDrTok/HobYuaps6rNvGGJO2ogn6jcBkEZkkIl6ck61PD9inFrgCQESm4gR9o7v+cnd9HjAPeCM2pUdnk78ZsP55Y0z6GjLoVTUE3Ak8A+zAGV2zXUTuE5Hr3d2+DHxGRLYAPweWqarijNbJF5HtOB8Yj6nq1uF4ISezqTaANzOD6RVFZ/PPGmNMwsiKZidVXYtzkjVy3b0Ry68DCwY5rh1niGXcVO9r4qLKIrKzMuNZhjHGxE1KXxnbFQyzrd4mMjPGpLeUDvrtDS30hHst6I0xaS2lg756n3Oh1Gy7R6wxJo2ldNDX+ANMKM2lrCA73qUYY0zcpGzQqyqbagPWbWOMSXspG/S1TR0cbu+xoDfGpL2UDfpj/fMW9MaYdJeyQV9TG6AgO4vJowriXYoxxsRVygb9Jn+AWRNKyMywicyMMektJYO+pTPIzoNtzLFhlcYYk5pBv3l/M6rWP2+MMZCiQV/jD5AhMHN8cbxLMcaYuEvRoG9iSnkh+dlRzdlmjDEpLeWCPhTuZXNts3XbGGOMK+WCfufBNo72hC3ojTHGlXJBv8lvF0oZY0yklAv6an+AUQXZVJbkxLsUY4xJCCkX9DV+ZyIzEbtQyhhjIMWC/mBrF3WBTuu2McaYCCkV9DXWP2+MMSdIuaDPzsrgwrFF8S7FGGMSRsoF/cWVxXizUuplGWPMGYkqEUVkiYjsFJE9IrJikO3jRWSdiLwqIltF5JqIbTNEZIOIbBeR10TEF8sXcExXMMz2hhZmW7eNMcYcZ8g5AkQkE/gu8F6gDtgoIk+r6usRu/0zsFpVvy8i04C1wEQRyQJ+CnxMVbeISCkQjPmrAFq7glxz0RgumzxyOJ7eGGOSVjSTwcwF9qjqWwAi8iRwAxAZ9AoUustFQIO7fBWwVVW3AKjqkVgUPZhRBT6+ffOs4Xp6Y4xJWtF03VQA+yMe17nrIn0duFVE6nBa8593158PqIg8IyKbROSrg/0BEVkuItUiUt3Y2HhaL8AYY8ypxeqs5S3Aj1S1ErgGeFxEMnC+Mbwb+Kj7+0YRuWLgwar6sKpWqWpVWVlZjEoyxhgD0QV9PTAu4nGluy7Sp4DVAKq6AfABI3Fa/+tV9bCqduC09mefadHGGGOiF03QbwQmi8gkEfECNwNPD9inFrgCQESm4gR9I/AMcJGI5LonZhdxfN++McaYYTbkyVhVDYnInTihnQmsUtXtInIfUK2qTwNfBh4RkS/inJhdpqoKBETkAZwPCwXWqurvhuvFGGOMOZE4eZw4qqqqtLq6Ot5lGGNMUhGRGlWtGmybXUJqjDEpzoLeGGNSXMJ13YhII+A/g6cYCRyOUTnJzt6L49n7cTx7P/qlwnsxQVUHHZ+ecEF/pkSk+mT9VOnG3ovj2ftxPHs/+qX6e2FdN8YYk+Is6I0xJsWlYtA/HO8CEoi9F8ez9+N49n70S+n3IuX66I0xxhwvFVv0xhhjIljQG2NMikuZoB/qdofpRETGubd2fN29heMX4l1TvIlIpnury9/Gu5Z4E5FiEVkjIm+IyA4RmR/vmuJJRL7o/jvZJiI/H67bncZTSgR9xO0OrwamAbe4tzRMVyHgy6o6DZgH3JHm7wfAF4Ad8S4iQXwb+IOqTgEuJo3fFxGpAO4CqlR1Os7EjTfHt6rYS4mgJ+J2h6raAxy73WFaUtW3VXWTu9yG8w954F3B0oaIVALXAo/Gu5Z4E5EiYCHwQwBV7VHV5vhWFXdZQI47lXou/bdCTRmpEvTR3O4wLYnIRGAW8Ep8K4mrB4GvAr3xLiQBTMK5V8RjblfWoyKSF++i4kVV64GVOPfUeBtoUdU/xreq2EuVoDeDEJF84JfA3araGu964kFErgMOqWpNvGtJEFk4d3n7vqrOAo4CaXtOS0RKcL79TwLGAnkicmt8q4q9VAn6aG53mFZExIMT8j9T1V/Fu544WgBcLyL7cLr0LheRn8a3pLiqA+pU9dg3vDWk9+09rwT2qmqjqgaBXwHvinNNMZcqQR/N7Q7ThogITh/sDlV9IN71xJOq3qOqlao6Eef/iz+pasq12KKlqgeA/SJygbvqCtL79p61wDz3dqeC836k3MnpIW8lmAxOdrvDOJcVTwuAjwGvichmd90/quraONZkEsfngZ+5jaK3gNviXE/cqOorIrIG2IQzWu1VUnA6BJsCwRhjUlyqdN0YY4w5CQt6Y4xJcRb0xhiT4izojTEmxVnQG2NMirOgN8aYFGdBb4wxKe7/A0q+r4+5ZcOfAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"tags":[],"needs_background":"light"}}]},{"cell_type":"markdown","metadata":{"id":"cta7Ez7I6E5a"},"source":["Fairly stable result even with smaller number of parameters"]},{"cell_type":"code","metadata":{"id":"9fx-z2Tv6E5a","executionInfo":{"status":"ok","timestamp":1604782784818,"user_tz":420,"elapsed":17155,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"1424ad0d-e900-4ca8-b44a-a67025b123b4","colab":{"base_uri":"https://localhost:8080/"}},"source":["results = model.evaluate(X_test, y_test)"],"execution_count":37,"outputs":[{"output_type":"stream","text":["313/313 [==============================] - 16s 51ms/step - loss: 0.0409 - accuracy: 0.9892\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"nV2F991l6E5c","executionInfo":{"status":"ok","timestamp":1604782784821,"user_tz":420,"elapsed":17151,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"outputId":"67eab0db-b234-4fa1-f1e2-88f1ceaf723b","colab":{"base_uri":"https://localhost:8080/"}},"source":["print('Test accuracy: ', results[1])"],"execution_count":38,"outputs":[{"output_type":"stream","text":["Test accuracy:  0.9891999959945679\n"],"name":"stdout"}]}]}